From dc37d07b80aff9238890aad989301b31338598af Mon Sep 17 00:00:00 2001 From: bjkim Date: Mon, 20 Oct 2025 15:43:59 +0900 Subject: [PATCH] =?UTF-8?q?[ADD]=20=EC=99=B8=EB=B6=80=20=EB=8D=B0=EC=9D=B4?= =?UTF-8?q?=ED=84=B0=EC=85=8B=20=EB=8B=A4=EC=9A=B4=EB=A1=9C=EB=93=9C=20?= =?UTF-8?q?=EB=B0=8F=20S3=20=EC=A0=80=EC=9E=A5=20API=20=EC=B6=94=EA=B0=80,?= =?UTF-8?q?=20Controller=20=EB=B0=8F=20Service=20=EB=A1=9C=EC=A7=81=20?= =?UTF-8?q?=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExternalDataSetController.java | 48 ++++-- .../etri/autoflow/service/DatasetService.java | 153 ++++++++++++++---- 2 files changed, 155 insertions(+), 46 deletions(-) diff --git a/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java b/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java index 749e66d..8924954 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java @@ -7,13 +7,18 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; +import kr.re.etri.autoflow.entity.MinioAttachmentEntity; import kr.re.etri.autoflow.service.DatasetService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import java.util.HashMap; import java.util.Map; +@Slf4j @Tag(name = "Dataset API", description = "외부 데이터셋 목록 조회 API (Python requests 코드 기반)") @RestController @RequestMapping("/api/datasets") @@ -35,26 +40,43 @@ public class ExternalDataSetController { public ResponseEntity getDatasetList( @Parameter(description = "프로젝트 인덱스", example = "10") @RequestParam(required = false) Integer ds_prj_idx, - @Parameter(description = "검색 키워드", example = "traffic") + @Parameter(description = "검색 키워드", example = "") @RequestParam(required = false) String search_keyword, - @Parameter(description = "그룹 이름", example = "AIGroup") + @Parameter(description = "그룹 이름", example = "") @RequestParam(required = false) String grp_name ) { Map result = datasetService.getDatasetList(ds_prj_idx, search_keyword, grp_name); return ResponseEntity.ok(result); } - @PostMapping("/download") - @Operation( - summary = "데이터셋 다운로드", - description = "외부 API 서버로 POST 요청을 보내 ZIP 파일을 다운로드합니다.", - parameters = { - @Parameter(name = "datasetName", description = "다운로드할 데이터셋 이름", in = ParameterIn.QUERY, required = true) - } - ) - public ResponseEntity downloadDataset( - @RequestParam("datasetName") String datasetName + @Operation(summary = "외부 데이터셋 다운로드 후 S3 저장", description = "Kubeflow 등 외부 API로부터 데이터셋을 다운로드하고 MinIO에 업로드합니다.") + @PostMapping("/dataset/save") + public ResponseEntity> saveDatasetToS3( + @RequestParam String datasetName, + @RequestParam(required = false) String path, + @RequestParam(required = false) Long refId, + @RequestParam(defaultValue = "DATASET") String refType, + @RequestParam(required = false) String title, + @RequestParam(required = false) String description, + @RequestParam(defaultValue = "1") Integer version, + @RequestParam String regUserId, + @RequestParam Long projectId ) { - return datasetService.downloadDataset(datasetName); + try { + MinioAttachmentEntity saved = datasetService.downloadDataset( + datasetName, path, refId, refType, title, description, version, regUserId, projectId + ); + + Map response = new HashMap<>(); + response.put("attachment", saved); + //response.put("minioUrl", minioAttachmentService.getFileUrl(saved.getStoragePath())); + + return ResponseEntity.ok(response); + } catch (Exception e) { + log.error("데이터셋 저장 실패", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(Map.of("error", e.getMessage())); + } } + } diff --git a/src/main/java/kr/re/etri/autoflow/service/DatasetService.java b/src/main/java/kr/re/etri/autoflow/service/DatasetService.java index fe7b448..af1a52b 100644 --- a/src/main/java/kr/re/etri/autoflow/service/DatasetService.java +++ b/src/main/java/kr/re/etri/autoflow/service/DatasetService.java @@ -1,17 +1,30 @@ package kr.re.etri.autoflow.service; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import kr.re.etri.autoflow.entity.MinioAttachmentEntity; +import kr.re.etri.autoflow.repository.MinioAttachmentRepository; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.core.io.buffer.DataBufferUtils; import org.springframework.http.*; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.util.UriComponentsBuilder; +import reactor.core.publisher.Flux; +import java.io.*; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; @Service @RequiredArgsConstructor @@ -20,9 +33,21 @@ public class DatasetService { private final RestTemplate restTemplate; + private final MinioAttachmentRepository minioAttachmentRepository; + + private static final String BASE_URL = "http://52.14.11.43:18010"; private static final String LIST_ENDPOINT = "/export/dataset_list"; + private final MinioClient minioClient; + + @Value("${minio.bucket}") + private String bucketName; + + @Value("${minio.endpoint}") + private String minioEndpoint; + + @SuppressWarnings("unchecked") public Map getDatasetList(Integer ds_prj_idx, String search_keyword, String grp_name) { try { @@ -86,48 +111,110 @@ public class DatasetService { .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) .build(); - public ResponseEntity downloadDataset(String datasetName) { + public MinioAttachmentEntity downloadDataset( + String datasetName, + String path, + Long refId, + String refType, + String title, + String description, + Integer version, + String regUserId, + Long projectId + ) { String apiUrl = baseUrl + DOWNLOAD_ENDPOINT; log.info("외부 API 요청 URL: {}", apiUrl); log.info("요청 데이터셋 이름: {}", datasetName); - // 요청 본문(JSON) - String requestBody = String.format("{\"dataset_name\":\"%s\"}", datasetName); + String storedName = UUID.randomUUID() + "-" + datasetName + ".zip"; + String objectName = (path == null || path.isEmpty()) ? storedName : path + "/" + storedName; try { - // WebClient로 외부 API POST 요청 - return webClient.post() + Flux dataBufferFlux = webClient.post() .uri(apiUrl) - .bodyValue(requestBody) - .exchangeToMono(response -> { - if (response.statusCode().equals(HttpStatus.OK)) { - // 파일 스트림 수신 - return response.bodyToMono(byte[].class) - .map(bytes -> { - ByteArrayResource resource = new ByteArrayResource(bytes); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); - headers.setContentDisposition(ContentDisposition.attachment() - .filename(datasetName + ".zip", StandardCharsets.UTF_8) - .build()); - return new ResponseEntity<>(resource, headers, HttpStatus.OK); - }); - } else { - // 오류 응답 처리 - return response.bodyToMono(String.class) - .map(errorBody -> { - log.error("다운로드 실패 (HTTP {}): {}", response.statusCode(), errorBody); - return ResponseEntity.status(response.statusCode()) - .contentType(MediaType.APPLICATION_JSON) - .body(errorBody); - }); + .bodyValue(Map.of("dataset_name", datasetName)) + .retrieve() + .bodyToFlux(DataBuffer.class) + .doOnError(e -> log.error("다운로드 중 오류 발생", e)); + + PipedOutputStream pos = new PipedOutputStream(); + PipedInputStream pis = new PipedInputStream(pos, 10 * 1024 * 1024); + + CountDownLatch latch = new CountDownLatch(1); + + // 파일 크기 저장용 + long[] totalBytes = {0}; + + Thread uploadThread = new Thread(() -> { + try { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .stream(pis, -1, 10 * 1024 * 1024) + .contentType("application/zip") + .build() + ); + log.info("MinIO 업로드 완료: {}", objectName); + } catch (Exception e) { + log.error("MinIO 업로드 실패", e); + throw new RuntimeException(e); + } finally { + try { pis.close(); } catch (Exception ignored) {} + latch.countDown(); + } + }); + uploadThread.start(); + + dataBufferFlux.subscribe( + buffer -> { + try { + byte[] bytes = new byte[buffer.readableByteCount()]; + buffer.read(bytes); + pos.write(bytes); + pos.flush(); + totalBytes[0] += bytes.length; + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + DataBufferUtils.release(buffer); } - }) - .block(); + }, + error -> { + log.error("다운로드 중 오류 발생", error); + try { pos.close(); } catch (Exception ignored) {} + latch.countDown(); + }, + () -> { + try { pos.close(); } catch (Exception ignored) {} + latch.countDown(); + log.info("다운로드 완료: {} MB", totalBytes[0] / (1024 * 1024)); + } + ); + + latch.await(); + + // DB 저장 시 size 컬럼 필수 + MinioAttachmentEntity attachment = MinioAttachmentEntity.builder() + .refId(refId) + .refType(refType) + .originalName(datasetName + ".zip") + .storedName(storedName) + .contentType("application/zip") + .storagePath(objectName) + .title(title != null ? title : datasetName) + .version(version) + .description(description) + .regUserId(regUserId) + .projectId(projectId) + .size(totalBytes[0]) + .build(); + + return minioAttachmentRepository.save(attachment); + } catch (Exception e) { - log.error("API 요청 중 오류 발생", e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body("API 요청 중 오류 발생: " + e.getMessage()); + log.error("외부 API 다운로드 및 MinIO 업로드 실패", e); + throw new RuntimeException("데이터셋 저장 실패: " + e.getMessage(), e); } } }