diff --git a/src/main/java/kr/re/etri/autoflow/controllers/DynamicMinioAttachmentController.java b/src/main/java/kr/re/etri/autoflow/controllers/DynamicMinioAttachmentController.java deleted file mode 100644 index e96d7da..0000000 --- a/src/main/java/kr/re/etri/autoflow/controllers/DynamicMinioAttachmentController.java +++ /dev/null @@ -1,172 +0,0 @@ -package kr.re.etri.autoflow.controllers; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import kr.re.etri.autoflow.entity.MinioAttachmentEntity; -import kr.re.etri.autoflow.payload.request.ProjectBaseSearchRequest; -import kr.re.etri.autoflow.service.DynamicMinioAttachmentService; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.domain.Page; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; - -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Slf4j -@RestController -@RequestMapping("/api/minio") -@RequiredArgsConstructor -public class DynamicMinioAttachmentController { - - private final DynamicMinioAttachmentService minioService; - - /** - * 파일 업로드 - */ - @PostMapping("/upload") - public ResponseEntity> uploadFile( - @RequestParam MultipartFile file, - @RequestParam(required = false) String path, - @RequestParam Long refId, - @RequestParam(defaultValue = "DATASET") String refType, - @RequestParam(required = false) String title, - @RequestParam(required = false) String description, - @RequestParam(defaultValue = "1") Integer version, - @RequestParam String regUserId, - @RequestParam Long projectId, - @RequestParam(defaultValue = "type1") String type - ) { - try { - MinioAttachmentEntity attachment = minioService.uploadFile( - file, path, refId, refType, title, description, version, regUserId, projectId, type - ); - - Map response = new HashMap<>(); - response.put("attachment", attachment); - - return ResponseEntity.ok(response); - } catch (Exception e) { - log.error("파일 업로드 실패", e); - return ResponseEntity.internalServerError() - .body(Map.of("error", e.getMessage())); - } - } - - /** - * 파일 다운로드 - */ - @GetMapping("/download") - public ResponseEntity downloadFile( - @RequestParam(defaultValue = "4/9d08fa7973cf4c39a0979bb4d70c640b/artifacts/sklearn-model/model.pkl") String objectName, - @RequestParam(defaultValue = "type1") String type - ) { - try { - byte[] bytes = minioService.downloadFile(objectName, type); - - String encodedFileName = URLEncoder.encode(objectName, StandardCharsets.UTF_8) - .replaceAll("\\+", "%20"); - - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFileName) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(bytes); - } catch (Exception e) { - log.error("파일 다운로드 실패: objectName={}, type={}", objectName, type, e); - String msg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); - return ResponseEntity.status(500) - .contentType(MediaType.APPLICATION_JSON) - .body(("{\"error\":\"" + msg.replace("\"", "\\\"") + "\"}").getBytes(java.nio.charset.StandardCharsets.UTF_8)); - } - } - - @PostMapping("/download-to-server") - @Operation( - summary = "MinIO 객체 서버 다운로드", - description = "MinIO에 저장된 객체를 서버의 지정된 경로로 다운로드합니다.", - responses = { - @ApiResponse(responseCode = "200", description = "파일 다운로드 성공"), - @ApiResponse(responseCode = "500", description = "파일 다운로드 실패") - } - ) - public String downloadFileToServer( - @Parameter(description = "다운로드할 MinIO 객체 이름", example="4/9d08fa7973cf4c39a0979bb4d70c640b/artifacts/sklearn-model/model.pkl", required = true) - @RequestParam String objectName, - - @Parameter(description = "MINIO 서버 타입 (type1[kubeflow],type2[mlflow])",example="type2", required = true) - @RequestParam String type, - - @Parameter(description = "서버에 저장할 로컬 경로", example="downloads/temp", required = false) - @RequestParam String localPath - ) { - try { - minioService.downloadFileToServer(objectName, type, localPath); - return "파일 다운로드 성공: " + localPath; - } catch (Exception e) { - log.error("서버 파일 다운로드 실패", e); - return "파일 다운로드 실패: " + e.getMessage(); - } - } - - /** - * YAML 텍스트 읽기 - */ - @GetMapping(value = "/readYamlText", produces = MediaType.TEXT_PLAIN_VALUE) - public ResponseEntity readYamlText( - @RequestParam String objectName, - @RequestParam(defaultValue = "type1") String type - ) { - try { - String content = minioService.readYamlText(objectName, type); - return ResponseEntity.ok(content); - } catch (Exception e) { - log.error("MinIO YAML 읽기 실패: {}", objectName, e); - return ResponseEntity.internalServerError() - .body("Error reading file: " + e.getMessage()); - } - } - - /** - * 파일 삭제 - */ - @DeleteMapping("/delete") - public ResponseEntity> deleteFile( - @RequestParam Long id, - @RequestParam(defaultValue = "type1") String type - ) { - boolean result = minioService.deleteFile(id, type); - return ResponseEntity.ok(Map.of("deleted", result)); - } - - /** - * 전체 조회 - */ - @GetMapping("/list") - public ResponseEntity> listAll() { - List list = minioService.findAll(); - return ResponseEntity.ok(list); - } - - /** - * 검색 + 페이지네이션 - */ - @PostMapping("/search") - public ResponseEntity> search( - @RequestBody ProjectBaseSearchRequest request, - @RequestParam String refType, - @RequestParam Integer refId - ) { - Page page = minioService.search(request, refType, refId); - return ResponseEntity.ok(page); - } -} diff --git a/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java b/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java index a9ad2bc..244e423 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java @@ -9,7 +9,7 @@ import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.annotation.PostConstruct; import kr.re.etri.autoflow.payload.request.EdgeSWVO; -import kr.re.etri.autoflow.service.DynamicMinioAttachmentService; +import kr.re.etri.autoflow.service.DynamicStorageAttachmentService; import kr.re.etri.autoflow.service.EdgeSWUploadService; import kr.re.etri.autoflow.service.ExternalAuthService; import lombok.RequiredArgsConstructor; @@ -45,7 +45,7 @@ public class ExternalAuthController { private final ExternalAuthService externalAuthService; private final EdgeSWUploadService edgeSWUploadService; - private final DynamicMinioAttachmentService minioService; + private final DynamicStorageAttachmentService minioService; private RestTemplate restTemplate; diff --git a/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java b/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java index fd880e4..33e1250 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java @@ -26,7 +26,7 @@ import java.util.Map; public class ExternalDataSetController { private final DatasetService datasetService; - private final kr.re.etri.autoflow.service.MinioAttachmentService minioAttachmentService; + private final kr.re.etri.autoflow.service.StorageAttachmentService minioAttachmentService; @Operation( summary = "데이터셋 목록 조회", diff --git a/src/main/java/kr/re/etri/autoflow/controllers/MinioAttachmentController.java b/src/main/java/kr/re/etri/autoflow/controllers/MinioAttachmentController.java deleted file mode 100644 index c0f6b6b..0000000 --- a/src/main/java/kr/re/etri/autoflow/controllers/MinioAttachmentController.java +++ /dev/null @@ -1,350 +0,0 @@ -package kr.re.etri.autoflow.controllers; - -import io.minio.DownloadObjectArgs; -import io.minio.GetObjectArgs; -import io.minio.MinioClient; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.tags.Tag; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.http.HttpServletResponse; -import kr.re.etri.autoflow.entity.MinioAttachmentEntity; -import kr.re.etri.autoflow.payload.request.ProjectBaseSearchRequest; -import kr.re.etri.autoflow.payload.request.ScriptMergeRequest; -import kr.re.etri.autoflow.service.MinioAttachmentService; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springdoc.core.annotations.ParameterObject; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.FileSystemResource; -import org.springframework.core.io.InputStreamResource; -import org.springframework.core.io.Resource; -import org.springframework.data.domain.Page; -import org.springframework.http.*; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; - -@RestController -@RequestMapping("/api/attachments") -@RequiredArgsConstructor -@Slf4j -@Tag(name = "첨부파일", description = "MinIO 첨부파일 관리 API") -public class MinioAttachmentController { - - private final MinioAttachmentService minioAttachmentService; - - private final MinioClient minioClient; - - @Value("${mlflow.url:}") - private String mlflowUrl; - @Value("${mlflow.user:}") - private String mlflowUser; - @Value("${mlflow.password:}") - private String mlflowPassword; - - @Operation(summary = "첨부파일 전체 조회") - @GetMapping - public ResponseEntity> getAll() { - return ResponseEntity.ok(minioAttachmentService.findAll()); - } - - @Operation(summary = "ID로 첨부파일 조회") - @GetMapping("/{id}") - public ResponseEntity getById( - @Parameter(description = "첨부파일 ID", required = true) - @PathVariable("id") Long id) { - return minioAttachmentService.findById(id) - .map(ResponseEntity::ok) - .orElse(ResponseEntity.notFound().build()); - } - - @Operation(summary = "검색 및 페이지네이션 첨부파일 목록 조회") - @GetMapping("/search") - public ResponseEntity> search( - @ParameterObject @ModelAttribute ProjectBaseSearchRequest request, - @Parameter( - description = "첨부파일 구분자. 예: WORKFLOW_STEP, DATASET, TRAINING_SCRIPT", - example = "WORKFLOW_STEP" - ) - @RequestParam(value = "refType", required = false) String refType, - @RequestParam(value = "refId", required = false, defaultValue = "0") Integer refId - ) { - Page page = minioAttachmentService.search(request, refType, refId); - return ResponseEntity.ok(page); - } - - @Operation(summary = "첨부파일 삭제 (MinIO 포함)") - @DeleteMapping("/{id}") - public ResponseEntity delete( - @Parameter(description = "첨부파일 ID", required = true) - @PathVariable("id") Long id) { - if (minioAttachmentService.delete(id)) { - return ResponseEntity.noContent().build(); - } - return ResponseEntity.notFound().build(); - } - - @Operation(summary = "파일 다운로드", description = "MinIO에서 파일을 다운로드합니다.") - @GetMapping("/download") - public ResponseEntity downloadFile(@RequestParam String objectName) { - try { - byte[] bytes = minioAttachmentService.downloadFile("mlpipeline", objectName); - - String encodedFileName = URLEncoder.encode(objectName, StandardCharsets.UTF_8) - .replaceAll("\\+", "%20"); // 공백 처리 - - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFileName) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(bytes); - - } catch (Exception e) { - log.error("파일 다운로드 실패", e); - return ResponseEntity.internalServerError().build(); - } - } - -// @GetMapping("/download_new") -// public ResponseEntity downloadFile_new(@RequestParam String objectName) { -// try { -// // MinIO에서 스트리밍으로 가져오기 -// InputStream is = minioClient.getObject( -// GetObjectArgs.builder() -// .bucket("mlpipeline") -// .object(objectName) -// .build() -// ); -// -// InputStreamResource resource = new InputStreamResource(is); -// -// String encodedFileName = URLEncoder.encode(objectName, StandardCharsets.UTF_8) -// .replaceAll("\\+", "%20"); -// -// return ResponseEntity.ok() -// .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encodedFileName) -// .contentType(MediaType.APPLICATION_OCTET_STREAM) -// .body(resource); -// -// } catch (Exception e) { -// log.error("파일 다운로드 실패", e); -// return ResponseEntity.internalServerError().build(); -// } -// } - - - @Operation(summary = "MinIO YAML 파일 읽기", description = "MinIO에서 YAML 파일을 다운로드하여 텍스트로 반환합니다.") - @GetMapping(value = "/readYamlText", produces = MediaType.TEXT_PLAIN_VALUE) - public ResponseEntity readYamlTextFromMinio(@RequestParam String objectName) { - try { - String content = minioAttachmentService.readYamlText("mlpipeline", objectName); - return ResponseEntity.ok(content); - } catch (Exception e) { - log.error("MinIO 파일 읽기 실패: {}", objectName, e); - return ResponseEntity.internalServerError() - .body("Error reading file: " + e.getMessage()); - } - } - - @Operation(summary = "MinIO 설정 조회", description = "파이프라인 YAML 생성 시 사용할 MinIO 설정(저장된 정보)을 반환합니다.") - @GetMapping("/minio-config") - public ResponseEntity> getMinioConfig() { - try { - Map config = new HashMap<>(); - config.put("minioEndpoint", nullToEmpty(minioAttachmentService.getMinioEndpoint())); - config.put("minioBucket", nullToEmpty(minioAttachmentService.getMinioBucket())); - config.put("minioAccessKey", nullToEmpty(minioAttachmentService.getMinioAccessKey())); - config.put("minioSecretKey", nullToEmpty(minioAttachmentService.getMinioSecretKey())); - config.put("minioEndpointPod", nullToEmpty(minioAttachmentService.getMinioEndpointPod())); - return ResponseEntity.ok(config); - } catch (Exception e) { - log.warn("MinIO 설정 조회 실패, 빈 값 반환: {}", e.getMessage()); - return ResponseEntity.ok(Map.of( - "minioEndpoint", "", "minioBucket", "", "minioAccessKey", "", "minioSecretKey", "", "minioEndpointPod", "" - )); - } - } - - private static String nullToEmpty(String s) { - return s != null ? s : ""; - } - - @Operation(summary = "MLflow 설정 조회", description = "Auto Script에서 MLflow 사용 시 YAML에 넣을 Tracking URI 등 설정을 반환합니다.") - @GetMapping("/mlflow-config") - public ResponseEntity> getMlflowConfig() { - try { - Map config = new HashMap<>(); - config.put("mlflowTrackingUri", nullToEmpty(mlflowUrl)); - config.put("mlflowUser", nullToEmpty(mlflowUser)); - config.put("mlflowPassword", nullToEmpty(mlflowPassword)); - return ResponseEntity.ok(config); - } catch (Exception e) { - log.warn("MLflow 설정 조회 실패, 빈 값 반환: {}", e.getMessage()); - return ResponseEntity.ok(Map.of( - "mlflowTrackingUri", "", "mlflowUser", "", "mlflowPassword", "" - )); - } - } - - @Operation(summary = "스크립트 컴파일 (py → KFP YAML)", description = "업로드된 .py 파일을 KFP에서 학습 실행용 파이프라인 YAML로 컴파일합니다. 서버에 Python3 및 kfp 패키지 필요.") - @PostMapping("/{id}/compile") - public ResponseEntity compilePyToYaml( - @Parameter(description = "첨부파일 ID (TRAINING_SCRIPT .py)", required = true) - @PathVariable("id") Long id) { - try { - Map result = minioAttachmentService.compilePyToKfpYaml(id); - return ResponseEntity.ok(result); - } catch (IllegalArgumentException e) { - log.warn("스크립트 컴파일 요청 오류: {}", e.getMessage()); - return ResponseEntity.badRequest().body(Map.of("error", e.getMessage())); - } catch (Exception e) { - log.error("스크립트 컴파일 실패: id={}", id, e); - String msg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(Map.of("error", msg)); - } - } - - @Operation(summary = "기존 컴파일 결과 조회", description = "첨부파일 ID 기준으로 이미 컴파일된 YAML이 있으면 MinIO 경로를 반환합니다.") - @GetMapping("/{id}/compiled-info") - public ResponseEntity getCompiledInfo( - @Parameter(description = "첨부파일 ID (TRAINING_SCRIPT .py)", required = true) - @PathVariable("id") Long id) { - try { - Optional pathOpt = minioAttachmentService.findCompiledYamlPath(id); - if (pathOpt.isEmpty()) { - return ResponseEntity.noContent().build(); - } - Map body = new HashMap<>(); - body.put("yamlStoragePath", pathOpt.get()); - body.put("objectName", pathOpt.get()); - return ResponseEntity.ok(body); - } catch (IllegalArgumentException e) { - log.warn("컴파일 정보 조회 요청 오류: {}", e.getMessage()); - return ResponseEntity.badRequest().body(Map.of("error", e.getMessage())); - } catch (Exception e) { - log.error("컴파일 정보 조회 실패: id={}", id, e); - String msg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(Map.of("error", msg)); - } - } - - @Operation(summary = "컴파일 번들 다운로드 (py + yaml ZIP)", description = "원본 .py와 컴파일된 YAML을 하나의 ZIP으로 다운로드합니다.") - @GetMapping("/download-compiled-bundle") - public ResponseEntity downloadCompiledBundle( - @RequestParam("id") Long attachmentId, - @RequestParam("yamlObjectName") String yamlObjectName) { - try { - byte[] zipBytes = minioAttachmentService.downloadCompiledBundle(attachmentId, yamlObjectName); - String baseName = yamlObjectName.contains("/") ? yamlObjectName.substring(yamlObjectName.lastIndexOf('/') + 1) : yamlObjectName; - String zipFileName = baseName.replaceAll("\\.yaml$", "") + "_bundle.zip"; - String encoded = URLEncoder.encode(zipFileName, StandardCharsets.UTF_8).replaceAll("\\+", "%20"); - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename*=UTF-8''" + encoded) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(zipBytes); - } catch (Exception e) { - log.error("컴파일 번들 다운로드 실패: id={}, yaml={}", attachmentId, yamlObjectName, e); - return ResponseEntity.badRequest().build(); - } - } - - @Operation(summary = "여러 Training Script를 머지하여 마스터 스크립트 생성", description = "여러 SCRIPT 첨부파일을 순서대로 실행하는 master.py를 생성합니다.") - @PostMapping("/merge-scripts") - public ResponseEntity> mergeScripts(@RequestBody ScriptMergeRequest request) { - try { - if (request.getScriptIds() == null || request.getScriptIds().isEmpty()) { - return ResponseEntity.badRequest().body(Map.of("error", "scriptIds 가 비었습니다.")); - } - if (request.getProjectId() == null || request.getRegUserId() == null) { - return ResponseEntity.badRequest().body(Map.of("error", "projectId, regUserId 는 필수입니다.")); - } - - List scripts = minioAttachmentService.findAllByIds(request.getScriptIds()); - - MinioAttachmentEntity master = minioAttachmentService.createMergedMaster(request, scripts); - - Map resp = new HashMap<>(); - resp.put("attachment", master); - resp.put("minioUrl", minioAttachmentService.getFileUrl(master.getStoragePath())); - return ResponseEntity.ok(resp); - } catch (Exception e) { - log.error("머지 스크립트 생성 실패", e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(Map.of("error", e.getMessage())); - } - } - - @Operation(summary = "파일 업로드", description = "MultipartFile을 MinIO 버킷에 업로드하고 DB에 기록합니다.") - @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity> uploadFile( - @Parameter(description = "업로드할 파일") @RequestPart("file") MultipartFile file, - @RequestPart(value = "path", required = false) String path, - @RequestParam(value = "refId", required = false) Long refId, - @RequestParam(value = "refType", required = false, defaultValue = "TRAINING_SCRIPT") String refType, - @RequestParam(value = "title", required = false) String title, - @RequestParam(value = "description", required = false) String description, - @RequestParam(value = "version", required = false, defaultValue = "1") Integer version, - @RequestParam(value = "regUserId") String regUserId, - @RequestParam(value = "projectId") Long projectId - - ) { - try { - MinioAttachmentEntity saved = minioAttachmentService.uploadFile( - file, path, refId, refType, title, description, version, regUserId, projectId - ); - - Map response = new HashMap<>(); - response.put("attachment", saved); - response.put("minioUrl", minioAttachmentService.getFileUrl(saved.getStoragePath())); - response.put("minioEndpoint", minioAttachmentService.getMinioEndpoint()); - response.put("minioBucket", minioAttachmentService.getMinioBucket()); - - return ResponseEntity.ok(response); - - } catch (Exception e) { - log.error("파일 업로드 실패", e); - String msg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(Map.of("error", msg)); - } - } - - @Operation(summary = "파일 업데이트", description = "파일을 새 버전으로 업로드합니다. 기존 파일은 그대로 보존되고, 버전은 +1 증가합니다.") - @PutMapping(value = "/{id}/update", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public ResponseEntity> updateFile( - @Parameter(description = "기존 첨부파일 ID", required = true) - @PathVariable("id") Long id, - @Parameter(description = "프로젝트 ID", required = true) - @RequestParam("projectId") Long projectId, - @Parameter(description = "새 파일") @RequestPart("file") MultipartFile file, - @RequestPart(value = "path", required = false) String path, - @RequestParam(value = "title", required = false) String title, - @RequestParam(value = "description", required = false) String description, - @RequestParam(value = "regUserId") String regUserId - ) { - try { - MinioAttachmentEntity updated = minioAttachmentService.updateFile( - id, projectId, file, path, title, description, regUserId - ); - - Map response = new HashMap<>(); - response.put("attachment", updated); - response.put("minioUrl", minioAttachmentService.getFileUrl(updated.getStoragePath())); - - return ResponseEntity.ok(response); - - } catch (Exception e) { - log.error("파일 업데이트 실패", e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } - } - -} \ No newline at end of file diff --git a/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java b/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java index 1fdef87..0930d86 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java @@ -6,10 +6,10 @@ import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; -import kr.re.etri.autoflow.entity.MinioAttachmentEntity; +import kr.re.etri.autoflow.entity.StorageAttachmentEntity; import kr.re.etri.autoflow.entity.WorkflowEntity; import kr.re.etri.autoflow.payload.request.CreateRunRequest; -import kr.re.etri.autoflow.service.MinioAttachmentService; +import kr.re.etri.autoflow.service.StorageAttachmentService; import kr.re.etri.autoflow.service.PipelineUploadService; import kr.re.etri.autoflow.service.WorkFlowService; import lombok.RequiredArgsConstructor; @@ -36,7 +36,7 @@ public class PipelineUploadController { private final PipelineUploadService pipelineUploadService; private final WorkFlowService workFlowService; - private final MinioAttachmentService minioAttachmentService; + private final StorageAttachmentService minioAttachmentService; @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResponseEntity> uploadPipeline( @@ -73,7 +73,7 @@ public class PipelineUploadController { workFlowService.save(workflow); // 2. MinIO 업로드 - MinioAttachmentEntity attachment = minioAttachmentService.uploadFile( + StorageAttachmentEntity attachment = minioAttachmentService.uploadFile( file, "workflows/" + projectId, workflow.getId(), diff --git a/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java b/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java index 68e237f..c6bc5a8 100644 --- a/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java +++ b/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java @@ -16,7 +16,8 @@ import org.hibernate.annotations.Comment; public class RefreshToken { @Id - @GeneratedValue(strategy = GenerationType.AUTO) + @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "refreshtoken_seq") + @SequenceGenerator(name = "refreshtoken_seq", sequenceName = "tb_refreshtoken_seq", allocationSize = 1) private long id; @OneToOne diff --git a/src/main/java/kr/re/etri/autoflow/service/AdminService.java b/src/main/java/kr/re/etri/autoflow/service/AdminService.java index db7b6f8..66b7d5f 100644 --- a/src/main/java/kr/re/etri/autoflow/service/AdminService.java +++ b/src/main/java/kr/re/etri/autoflow/service/AdminService.java @@ -45,7 +45,6 @@ public class AdminService { private static final Pattern WAIT_LOG_ARCHIVE_KEY = Pattern.compile("\\bkey:\\s*([\\w\\-./]+/main\\.log)\\b"); - private final MinioAttachmentService minioAttachmentService; private final RestTemplate restTemplate; private final KubernetesPodHealthService podHealthService; private final PipelineUploadService pipelineUploadService; @@ -55,6 +54,13 @@ public class AdminService { private String kubeflowUrl; @Value("${mlflow.url:}") private String mlflowUrl; + @Value("${minio.endpoint:}") + private String minioEndpoint; + @Value("${minio.access-key:}") + private String minioAccessKey; + @Value("${minio.secret-key:}") + private String minioSecretKey; + /** true면 KFP ml-pipeline v1beta1 노드 로그 API를 kubectl보다 먼저 시도 (클러스터 내부 조회, UI와 동일 경로) */ @Value("${admin.k8s.prefer-kfp-api-for-logs:true}") private boolean preferKfpApiForLogs; @@ -160,15 +166,15 @@ public class AdminService { */ private Map checkMinio() { Map out = new HashMap<>(); - String endpoint = (minioAttachmentService.getMinioEndpoint() != null - ? minioAttachmentService.getMinioEndpoint().replaceAll("/+$", "") : "").trim(); + String endpoint = (minioEndpoint != null + ? minioEndpoint.replaceAll("/+$", "") : "").trim(); if (endpoint.isBlank()) { out.put("status", "skip"); out.put("message", "MinIO endpoint 미설정"); return out; } - String accessKey = minioAttachmentService.getMinioAccessKey(); - String secretKey = minioAttachmentService.getMinioSecretKey(); + String accessKey = minioAccessKey; + String secretKey = minioSecretKey; if (accessKey == null) accessKey = ""; if (secretKey == null) secretKey = ""; try { @@ -611,12 +617,13 @@ public class AdminService { return null; } try { - byte[] bytes = minioAttachmentService.downloadFile(KFP_ARCHIVE_BUCKET, key.trim()); - String body = new String(bytes, StandardCharsets.UTF_8); - if (body.isBlank()) { - return null; + MinioClient client = MinioClient.builder().endpoint(minioEndpoint).credentials(minioAccessKey, minioSecretKey).build(); + try (java.io.InputStream is = client.getObject(io.minio.GetObjectArgs.builder().bucket(KFP_ARCHIVE_BUCKET).object(key.trim()).build())) { + byte[] bytes = is.readAllBytes(); + String body = new String(bytes, StandardCharsets.UTF_8); + if (body.isBlank()) return null; + return "-- MinIO archived main.log (bucket=" + KFP_ARCHIVE_BUCKET + ", key=" + key.trim() + ") --\n\n" + body; } - return "-- MinIO archived main.log (bucket=" + KFP_ARCHIVE_BUCKET + ", key=" + key.trim() + ") --\n\n" + body; } catch (Exception e) { return "-- MinIO archived main.log 읽기 실패 (bucket=" + KFP_ARCHIVE_BUCKET + ", key=" + key.trim() + "): " + (e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName()) diff --git a/src/main/java/kr/re/etri/autoflow/service/DynamicMinioAttachmentService.java b/src/main/java/kr/re/etri/autoflow/service/DynamicMinioAttachmentService.java deleted file mode 100644 index 49a3cf0..0000000 --- a/src/main/java/kr/re/etri/autoflow/service/DynamicMinioAttachmentService.java +++ /dev/null @@ -1,286 +0,0 @@ -package kr.re.etri.autoflow.service; - -import io.minio.GetObjectArgs; -import io.minio.MinioClient; -import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; -import jakarta.transaction.Transactional; -import kr.re.etri.autoflow.config.MinioTypeProperties; -import kr.re.etri.autoflow.entity.MinioAttachmentEntity; -import kr.re.etri.autoflow.payload.request.ProjectBaseSearchRequest; -import kr.re.etri.autoflow.repository.MinioAttachmentRepository; -import kr.re.etri.autoflow.specification.MinioAttachmentSpecification; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.data.domain.*; -import org.springframework.data.jpa.domain.Specification; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -@Slf4j -@Service -@RequiredArgsConstructor -@Transactional -public class DynamicMinioAttachmentService { - - private final MinioAttachmentRepository minioAttachmentRepository; - private final MinioAttachmentSpecification minioAttachmentSpecification; - private final MinioTypeProperties minioTypeProperties; - private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - /** MinioClient 생성 (type 기반, 설정에서 로드) */ - private MinioClient getClientByType(String type) { - MinioTypeProperties.TypeConfig config = minioTypeProperties.getByType(type); - return MinioClient.builder() - .endpoint(config.getEndpoint()) - .credentials(config.getAccessKey(), config.getSecretKey()) - .build(); - } - - /** Bucket 조회 (type 기반) */ - private String getBucketByType(String type) { - return minioTypeProperties.getByType(type).getBucket(); - } - - /** 파일 업로드 & DB 저장 */ - public MinioAttachmentEntity uploadFile(MultipartFile file, - String path, - Long refId, - String refType, - String title, - String description, - Integer version, - String regUserId, - Long projectId, - String type - ) throws Exception { - MinioClient client = getClientByType(type); - String bucketName = getBucketByType(type); - - try (InputStream is = file.getInputStream()) { - String storedName = UUID.randomUUID() + "-" + file.getOriginalFilename(); - String objectName = (path == null || path.isEmpty()) - ? storedName - : path + "/" + storedName; - - // MinIO 업로드 - client.putObject( - PutObjectArgs.builder() - .bucket(bucketName) - .object(objectName) - .stream(is, is.available(), -1) - .contentType(file.getContentType()) - .build() - ); - - // DB 저장 - MinioAttachmentEntity attachment = MinioAttachmentEntity.builder() - .refId(refId) - .refType(refType) - .originalName(file.getOriginalFilename()) - .storedName(storedName) - .contentType(file.getContentType()) - .size(file.getSize()) - .storagePath(objectName) - .title(title != null ? title : file.getOriginalFilename()) - .version(version) - .description(description) - .regUserId(regUserId) - .projectId(projectId) - .build(); - - return minioAttachmentRepository.save(attachment); - } - } - - /** 파일 다운로드 */ - public byte[] downloadFile(String objectName, String type) { - MinioClient client = getClientByType(type); - String bucketName = getBucketByType(type); - - try { - // 1. mlflow-artifacts:/ 접두어 제거 - String cleanObjectName = objectName.replaceFirst("^mlflow-artifacts:/", ""); - - // 2. 잘못된 슬래시 제거 - cleanObjectName = cleanObjectName.replaceAll("^/+", "").replaceAll("/+$", ""); - - // 3. 파일 확장자가 없는 경우 (디렉터리 요청으로 추정) - // → MLflow 구조상 실제 파일은 artifacts/ 하위에 있으므로 경로 자동 보정 - if (!cleanObjectName.matches(".*\\.[a-zA-Z0-9]+$")) { - throw new RuntimeException("요청된 객체가 파일이 아닙니다. 실제 파일 경로를 포함해야 합니다: " + cleanObjectName); - } - - try (InputStream is = client.getObject( - GetObjectArgs.builder() - .bucket(bucketName) - .object(cleanObjectName) - .build() - )) { - return is.readAllBytes(); - } - - } catch (io.minio.errors.ErrorResponseException e) { - throw new RuntimeException( - "MinIO 서버가 요청을 거부했습니다: " + objectName + - ", 코드=" + e.errorResponse().code() + - ", 버킷이름=" + bucketName + - ", 메시지=" + e.errorResponse().message() + - ", 요청ID=" + e.errorResponse().requestId() + - ", 호스트ID=" + e.errorResponse().hostId(), e); - - } catch (Exception e) { - throw new RuntimeException("MinIO 파일 다운로드 실패: " + objectName, e); - } - } - - /** - * MinIO에서 파일을 다운로드하여 서버 로컬에 저장 - * @param objectName MinIO 객체 경로 - * @param type MinIO 타입 - * @param localPath 서버에 저장할 로컬 경로 (예: downloads/temp) - * @return 저장된 로컬 파일 - */ - public File downloadFileToServer(String objectName, String type, String localPath) { - MinioClient client = getClientByType(type); - String bucketName = getBucketByType(type); - - try { - // mlflow-artifacts:/ 접두어 제거 - String cleanObjectName = objectName.replaceFirst("^mlflow-artifacts:/", "") - .replaceAll("^/+", "").replaceAll("/+$", ""); - - // 파일 확장자 체크 - if (!cleanObjectName.matches(".*\\.[a-zA-Z0-9]+$")) { - throw new RuntimeException("요청된 객체가 파일이 아닙니다: " + cleanObjectName); - } - - // 파일명 추출 - String fileName = Paths.get(cleanObjectName).getFileName().toString(); - File localDir = new File(localPath); - if (!localDir.exists()) localDir.mkdirs(); - File localFile = new File(localDir, fileName); - - try (InputStream is = client.getObject( - GetObjectArgs.builder() - .bucket(bucketName) - .object(cleanObjectName) - .build() - ); - FileOutputStream fos = new FileOutputStream(localFile) - ) { - byte[] buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = is.read(buffer)) != -1) { - fos.write(buffer, 0, bytesRead); - } - } - - return localFile; - - } catch (Exception e) { - throw new RuntimeException("MinIO 파일 다운로드 실패: " + objectName + ". 원인: " + e.getMessage(), e); - } - } - - - /** YAML 텍스트 읽기 */ - public String readYamlText(String objectName, String type) { - MinioClient client = getClientByType(type); - String bucketName = getBucketByType(type); - - try (InputStream is = client.getObject( - GetObjectArgs.builder().bucket(bucketName).object(objectName).build() - )) { - return new String(is.readAllBytes(), StandardCharsets.UTF_8); - } catch (Exception e) { - throw new RuntimeException("MinIO YAML 읽기 실패: " + objectName, e); - } - } - - /** 파일 삭제 */ - public boolean deleteFile(Long id, String type) { - Optional attachmentOpt = minioAttachmentRepository.findById(id); - if (attachmentOpt.isEmpty()) return false; - - MinioAttachmentEntity attachment = attachmentOpt.get(); - MinioClient client = getClientByType(type); - String bucketName = getBucketByType(type); - - try { - client.removeObject( - RemoveObjectArgs.builder() - .bucket(bucketName) - .object(attachment.getStoragePath()) - .build() - ); - minioAttachmentRepository.deleteById(id); - return true; - } catch (Exception e) { - log.error("MinIO 파일 삭제 실패: " + attachment.getStoragePath(), e); - return false; - } - } - - // 이하 기존 search, findById, update 등 기존 DB 관련 메서드는 그대로 재사용 가능 - public List findAll() { - return minioAttachmentRepository.findAll(); - } - - public Optional findById(Long id) { - return minioAttachmentRepository.findById(id); - } - - public Page search(ProjectBaseSearchRequest request, String refType, Integer refId) { - int pageIndex = request.getPage() > 0 ? request.getPage() - 1 : 0; - - Pageable pageable = PageRequest.of( - pageIndex, - request.getSize(), - Sort.by(Sort.Direction.fromString(request.getSortDirection()), request.getSortField()) - ); - - LocalDate startDate = parseDate(request.getStartDate()); - LocalDate endDate = parseDate(request.getEndDate()); - - Specification spec = - minioAttachmentSpecification.searchByConditions( - refType, - refId, - request.getSearchType(), - request.getKeyword(), - startDate, - endDate - ); - - if (request.getProjectId() != null) { - spec = spec.and((root, query, cb) -> - cb.equal(root.get("projectId"), request.getProjectId()) - ); - } - - return minioAttachmentRepository.findAll(spec, pageable); - } - - private LocalDate parseDate(String dateStr) { - if (dateStr == null || dateStr.isBlank()) return null; - try { - return LocalDate.parse(dateStr, formatter); - } catch (DateTimeParseException e) { - throw new IllegalArgumentException("날짜 형식이 잘못되었습니다. yyyy-MM-dd 형식이어야 합니다: " + dateStr); - } - } -} diff --git a/src/main/java/kr/re/etri/autoflow/service/MinioAttachmentService.java b/src/main/java/kr/re/etri/autoflow/service/MinioAttachmentService.java deleted file mode 100644 index 519aa48..0000000 --- a/src/main/java/kr/re/etri/autoflow/service/MinioAttachmentService.java +++ /dev/null @@ -1,684 +0,0 @@ -package kr.re.etri.autoflow.service; - -import io.minio.GetObjectArgs; -import io.minio.MinioClient; -import io.minio.PutObjectArgs; -import io.minio.StatObjectArgs; -import io.minio.RemoveObjectArgs; -import jakarta.transaction.Transactional; -import kr.re.etri.autoflow.entity.MinioAttachmentEntity; -import kr.re.etri.autoflow.payload.request.BaseSearchRequest; -import kr.re.etri.autoflow.payload.request.ProjectBaseSearchRequest; -import kr.re.etri.autoflow.payload.request.ScriptMergeRequest; -import kr.re.etri.autoflow.repository.MinioAttachmentRepository; -import kr.re.etri.autoflow.specification.MinioAttachmentSpecification; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.*; -import org.springframework.data.jpa.domain.Specification; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.io.ByteArrayOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -@Slf4j -@Service -@RequiredArgsConstructor -@Transactional -public class MinioAttachmentService { - - private final MinioClient minioClient; - private final MinioAttachmentRepository minioAttachmentRepository; - private final MinioAttachmentSpecification minioAttachmentSpecification; - - @Value("${minio.bucket}") - private String bucketName; - - @Value("${minio.endpoint}") - private String minioEndpoint; - - @Value("${minio.access-key:}") - private String minioAccessKey; - - @Value("${minio.secret-key:}") - private String minioSecretKey; - - /** Pod에서 접근할 MinIO 주소 (비어 있으면 클라이언트에서 직접 입력) */ - @Value("${minio.endpoint.pod:}") - private String minioEndpointPod; - - /** KFP py→YAML 컴파일 시 사용할 Python 실행 파일 (Windows는 python, Linux는 python3 권장) */ - @Value("${kfp.compile.python-command:python3}") - private String pythonCommand; - - private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - /** - * 파일 업로드 & DB 저장 - */ - public MinioAttachmentEntity uploadFile(MultipartFile file, - String path, - Long refId, - String refType, - String title, - String description, - Integer version, - String regUserId, - Long projectId - ) throws Exception { - - try (InputStream is = file.getInputStream()) { - String storedName = UUID.randomUUID() + "-" + file.getOriginalFilename(); - String objectName = (path == null || path.isEmpty()) - ? storedName - : path + "/" + storedName; - - // MinIO 업로드 - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucketName) - .object(objectName) - .stream(is, is.available(), -1) - .contentType(file.getContentType()) - .build() - ); - - // DB 저장 - MinioAttachmentEntity attachment = MinioAttachmentEntity.builder() - .refId(refId) - .refType(refType) - .originalName(file.getOriginalFilename()) - .storedName(storedName) - .contentType(file.getContentType()) - .size(file.getSize()) - .storagePath(objectName) - .title(title != null ? title : file.getOriginalFilename()) - .version(version) - .description(description) - .regUserId(regUserId) - .projectId(projectId) - .build(); - - return minioAttachmentRepository.save(attachment); - } - } - - /** - * 전체 조회 - */ - public List findAll() { - return minioAttachmentRepository.findAll(); - } - - /** - * ID 조회 - */ - public Optional findById(Long id) { - return minioAttachmentRepository.findById(id); - } - - /** - * 검색 + 페이지네이션 - */ - public Page search(ProjectBaseSearchRequest request, String refType, Integer refId) { - int pageIndex = request.getPage() > 0 ? request.getPage() - 1 : 0; - - Pageable pageable = PageRequest.of( - pageIndex, - request.getSize(), - Sort.by(Sort.Direction.fromString(request.getSortDirection()), request.getSortField()) - ); - - LocalDate startDate = parseDate(request.getStartDate()); - LocalDate endDate = parseDate(request.getEndDate()); - - Specification spec = - minioAttachmentSpecification.searchByConditions( - refType, - refId, - request.getSearchType(), - request.getKeyword(), - startDate, - endDate - ); - - if (request.getProjectId() != null) { - spec = spec.and((root, query, cb) -> - cb.equal(root.get("projectId"), request.getProjectId()) - ); - } - - return minioAttachmentRepository.findAll(spec, pageable); - } - - public List findAllByIds(List ids) { - return minioAttachmentRepository.findAllById(ids); - } - - private LocalDate parseDate(String dateStr) { - if (dateStr == null || dateStr.isBlank()) return null; - try { - return LocalDate.parse(dateStr, formatter); - } catch (DateTimeParseException e) { - throw new IllegalArgumentException("날짜 형식이 잘못되었습니다. yyyy-MM-dd 형식이어야 합니다: " + dateStr); - } - } - - public byte[] downloadFile(String bucketName, String objectName) { - try (InputStream is = minioClient.getObject( - GetObjectArgs.builder().bucket(bucketName).object(objectName).build() - )) { - return is.readAllBytes(); - } catch (Exception e) { - throw new RuntimeException("MinIO 파일 다운로드 실패: " + objectName, e); - } - } - - - // YAML 텍스트 읽기 - public String readYamlText(String bucketName, String objectName) { - try (InputStream is = minioClient.getObject( - GetObjectArgs.builder().bucket(bucketName).object(objectName).build() - )) { - return new String(is.readAllBytes(), StandardCharsets.UTF_8); - } catch (Exception e) { - throw new RuntimeException("MinIO YAML 읽기 실패: " + objectName, e); - } - } - - public MinioAttachmentEntity updateFile( - Long id, - Long projectId, // 추가 - MultipartFile file, - String path, - String title, - String description, - String regUserId - ) throws Exception { - // 기존 엔티티 조회 - MinioAttachmentEntity existing = minioAttachmentRepository.findById(id) - .orElseThrow(() -> new IllegalArgumentException("첨부파일을 찾을 수 없습니다. ID=" + id)); - - // 최신 버전 조회 - Integer latestVersion = minioAttachmentRepository - .findTopByRefIdAndRefTypeOrderByVersionDesc(existing.getRefId(), existing.getRefType()) - .map(MinioAttachmentEntity::getVersion) - .orElse(0); - - int newVersion = latestVersion + 1; - - // 새 파일 업로드 - String storedName = UUID.randomUUID() + "-" + file.getOriginalFilename(); - String objectName = (path == null || path.isEmpty()) - ? storedName - : path + "/" + storedName; - - try (InputStream is = file.getInputStream()) { - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucketName) - .object(objectName) - .stream(is, is.available(), -1) - .contentType(file.getContentType()) - .build() - ); - } - - // 새로운 엔티티 생성 (이전 데이터는 그대로 두고, 새로운 버전 생성) - MinioAttachmentEntity newAttachment = MinioAttachmentEntity.builder() - .projectId(projectId) // 추가 - .refId(existing.getRefId()) - .refType(existing.getRefType()) - .originalName(file.getOriginalFilename()) - .storedName(storedName) - .contentType(file.getContentType()) - .size(file.getSize()) - .storagePath(objectName) - .title(title != null ? title : existing.getTitle()) - .description(description != null ? description : existing.getDescription()) - .version(newVersion) - .regUserId(regUserId) - .build(); - - return minioAttachmentRepository.save(newAttachment); - } - - /** - * 생성 (DB만 저장, 파일은 따로 업로드된 경우) - */ - public MinioAttachmentEntity create(MinioAttachmentEntity entity) { - return minioAttachmentRepository.save(entity); - } - - /** - * 업데이트 - */ - public Optional update(Long id, MinioAttachmentEntity dto) { - return minioAttachmentRepository.findById(id) - .map(entity -> { - BeanUtils.copyProperties(dto, entity, "id", "regDt"); // ID, 등록일 제외 - return minioAttachmentRepository.save(entity); - }); - } - - /** - * 삭제 - */ - @Transactional - public boolean delete(Long id) { - Optional attachmentOpt = minioAttachmentRepository.findById(id); - if (attachmentOpt.isEmpty()) { - return false; - } - - MinioAttachmentEntity attachment = attachmentOpt.get(); - try { - // MinIO 파일 삭제 - minioClient.removeObject( - RemoveObjectArgs.builder() - .bucket(bucketName) - .object(attachment.getStoragePath()) - .build() - ); - - // DB에서 삭제 - minioAttachmentRepository.deleteById(id); - return true; - - } catch (Exception e) { - log.error("MinIO 파일 삭제 실패: " + attachment.getStoragePath(), e); - return false; - } - } - - /** - * MinIO URL 반환 - */ - public String getFileUrl(String objectName) { - return String.format("%s/%s/%s", minioEndpoint, bucketName, objectName); - } - - /** 파이프라인 Pod에서 MinIO 직접 접근 시 사용 (endpoint) */ - public String getMinioEndpoint() { - return minioEndpoint; - } - - /** 파이프라인 Pod에서 MinIO 직접 접근 시 사용 (bucket) */ - public String getMinioBucket() { - return bucketName; - } - - public String getMinioAccessKey() { - return minioAccessKey; - } - - public String getMinioSecretKey() { - return minioSecretKey; - } - - /** Pod에서 접근할 MinIO endpoint (설정 시 YAML에 반영) */ - public String getMinioEndpointPod() { - return minioEndpointPod != null ? minioEndpointPod : ""; - } - - private String buildMasterScript(List scripts, String pipelineName) { - // 파이프라인 본문(각 스텝)을 생성 - StringBuilder body = new StringBuilder(); - String prevOutput = "\"\""; // 첫 스텝은 input_dir = "" - - for (int i = 0; i < scripts.size(); i++) { - MinioAttachmentEntity att = scripts.get(i); - String safeTitle = att.getTitle() != null ? att.getTitle() : att.getOriginalName(); - safeTitle = safeTitle.replace("\"", "\\\""); // 파이썬 문자열용 이스케이프 - - int stepIndex = i + 1; - String stepVar = "step" + stepIndex; - String outputDir = "/workspace/outputs/step_" + stepIndex; - - body.append(" ").append(stepVar).append(" = run_script_component(\n"); - body.append(" minio_endpoint_pod=minio_endpoint_pod,\n"); - body.append(" minio_bucket=minio_bucket,\n"); - body.append(" minio_access_key=minio_access_key,\n"); - body.append(" minio_secret_key=minio_secret_key,\n"); - body.append(" object_name=\"").append(att.getStoragePath().replace("\"", "\\\"")).append("\",\n"); - body.append(" step_name=\"step-").append(stepIndex).append("-").append(safeTitle).append("\",\n"); - body.append(" input_dir=").append(prevOutput).append(",\n"); - body.append(" output_dir=\"").append(outputDir).append("\",\n"); - body.append(" )\n\n"); - - prevOutput = stepVar + ".output"; - } - - String endpointPod = getMinioEndpointPod(); - String bucket = getMinioBucket(); - String accessKey = getMinioAccessKey(); - String secretKey = getMinioSecretKey(); - if (endpointPod == null) { - endpointPod = ""; - } - if (bucket == null) { - bucket = "mlpipeline"; - } - if (accessKey == null) { - accessKey = ""; - } - if (secretKey == null) { - secretKey = ""; - } - // 공백/개행 제거 및 끝 슬래시 제거 - endpointPod = endpointPod.trim() - .replaceFirst("^https?://", "") - .replaceAll("/+$", ""); - - String template = """ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -from kfp import dsl - - -@dsl.component(base_image="python:3.10") -def run_script_component( - minio_endpoint_pod: str, - minio_bucket: str, - minio_access_key: str, - minio_secret_key: str, - object_name: str, - step_name: str, - input_dir: str = "", - output_dir: str = "/workspace/outputs", -) -> str: - import os - import subprocess - import sys - - # 파라미터 정규화 - minio_endpoint_pod = (minio_endpoint_pod or "").strip().rstrip("/") - minio_bucket = (minio_bucket or "").strip().strip("/") - - os.makedirs(output_dir, exist_ok=True) - - # MinIO Python SDK 설치 - print("[MASTER] installing minio client...") - subprocess.run([sys.executable, "-m", "pip", "install", "minio"], check=True) - - from minio import Minio - - # 스크립트 파일을 MinIO에서 로컬로 다운로드 - client = Minio( - minio_endpoint_pod, - access_key=minio_access_key, - secret_key=minio_secret_key, - secure=False, - ) - client.fget_object(minio_bucket, object_name, "script.py") - - # 사용자 스크립트 실행 - cmd_parts = [] - if input_dir: - cmd_parts.append(f"export INPUT_DIR={input_dir}") - cmd_parts.append(f"python script.py --input_dir \\"$INPUT_DIR\\" --output_dir \\"{output_dir}\\"") - cmd = " && ".join(cmd_parts) - - print(f"[MASTER] STEP {step_name} 실행: {cmd}") - result = subprocess.run(cmd, shell=True) - if result.returncode != 0: - raise RuntimeError(f"STEP {step_name} 실패 (exit={result.returncode})") - - print(f"[MASTER] STEP {step_name} 완료, output_dir={output_dir}") - return output_dir - - -@dsl.pipeline(name="%s") -def pipeline( - mlflow_experiment_name: str = "", - minio_endpoint_pod: str = "%s", - minio_bucket: str = "%s", - minio_access_key: str = "%s", - minio_secret_key: str = "%s", -): -%s -"""; - - return String.format( - template, - pipelineName.replace("\"", "\\\""), - endpointPod.replace("\\", "\\\\").replace("\"", "\\\""), - bucket.replace("\\", "\\\\").replace("\"", "\\\""), - accessKey.replace("\\", "\\\\").replace("\"", "\\\""), - secretKey.replace("\\", "\\\\").replace("\"", "\\\""), - body.toString() - ); - } - - public MinioAttachmentEntity createMergedMaster(ScriptMergeRequest req, List scripts) throws Exception { - if (scripts == null || scripts.isEmpty()) { - throw new IllegalArgumentException("머지할 스크립트가 없습니다."); - } - - String pipelineName = (req.getTitle() != null && !req.getTitle().isBlank()) - ? req.getTitle() - : "merged-training-script"; - - String masterPy = buildMasterScript(scripts, pipelineName); - byte[] bytes = masterPy.getBytes(StandardCharsets.UTF_8); - - String originalName = (req.getTitle() != null && !req.getTitle().isBlank()) - ? req.getTitle() + ".py" - : "merged_pipeline.py"; - - String path = "scripts/merged"; - Long refId = req.getRefId() != null ? req.getRefId() : 0L; - String refType = (req.getRefType() != null && !req.getRefType().isBlank()) - ? req.getRefType() - : "TRAINING_SCRIPT"; - String title = req.getTitle() != null ? req.getTitle() : "Merged Script"; - String description = req.getDescription(); - Integer version = 1; - String regUserId = req.getRegUserId(); - Long projectId = req.getProjectId(); - - // uploadFile 과 동일한 로직을 bytes 기반으로 수행 - String storedName = UUID.randomUUID() + "-" + originalName; - String objectName = (path == null || path.isEmpty()) - ? storedName - : path + "/" + storedName; - - try (InputStream is = new ByteArrayInputStream(bytes)) { - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucketName) - .object(objectName) - .stream(is, bytes.length, -1) - .contentType("text/x-python") - .build() - ); - } - - MinioAttachmentEntity attachment = MinioAttachmentEntity.builder() - .refId(refId) - .refType(refType) - .originalName(originalName) - .storedName(storedName) - .contentType("text/x-python") - .size((long) bytes.length) - .storagePath(objectName) - .title(title) - .version(version) - .description(description) - .regUserId(regUserId) - .projectId(projectId) - .build(); - - return minioAttachmentRepository.save(attachment); - } - - /** - * 업로드된 .py 파일을 KFP에서 학습 실행용 파이프라인 YAML로 컴파일합니다. - * 서버에 Python 3 및 kfp 패키지(pip install kfp)가 설치되어 있어야 합니다. - * 스크립트에는 @dsl.pipeline 데코레이터가 붙은 함수가 필요하며, 함수 이름은 'pipeline' 또는 'my_pipeline'을 권장합니다. - * - * @return yamlStoragePath(MinIO 객체 경로), yamlContent(생성된 YAML 본문) - */ - public Map compilePyToKfpYaml(Long id) throws Exception { - MinioAttachmentEntity attachment = minioAttachmentRepository.findById(id) - .orElseThrow(() -> new IllegalArgumentException("첨부파일을 찾을 수 없습니다. ID=" + id)); - - String originalName = attachment.getOriginalName(); - if (originalName == null || !originalName.toLowerCase().endsWith(".py")) { - throw new IllegalArgumentException("Python(.py) 파일만 컴파일할 수 있습니다. 파일: " + originalName); - } - - String pyContent = readYamlText(bucketName, attachment.getStoragePath()); - Path tempDir = Files.createTempDirectory("kfp_compile_"); - try { - Path pyPath = tempDir.resolve("pipeline.py"); - Files.writeString(pyPath, pyContent, StandardCharsets.UTF_8); - Path yamlPath = tempDir.resolve("output.yaml"); - - Path helperPath = tempDir.resolve("compile_kfp_pipeline.py"); - try (InputStream is = getClass().getResourceAsStream("/scripts/compile_kfp_pipeline.py")) { - if (is == null) { - throw new IllegalStateException("컴파일 헬퍼 스크립트를 찾을 수 없습니다. (scripts/compile_kfp_pipeline.py)"); - } - Files.copy(is, helperPath); - } - - ProcessBuilder pb = new ProcessBuilder( - pythonCommand != null && !pythonCommand.isBlank() ? pythonCommand : "python3", - helperPath.toAbsolutePath().toString(), - pyPath.toAbsolutePath().toString(), - yamlPath.toAbsolutePath().toString() - ); - pb.redirectErrorStream(true); - Process p = pb.start(); - StringBuilder out = new StringBuilder(); - try (BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8))) { - String line; - while ((line = r.readLine()) != null) { - out.append(line).append("\n"); - } - } - int exit = p.waitFor(); - if (exit != 0) { - log.warn("KFP 컴파일 stderr/stdout: {}", out); - throw new RuntimeException("스크립트 컴파일 실패: " + (out.length() > 0 ? out.toString().trim() : "exit " + exit)); - } - - if (!Files.exists(yamlPath)) { - throw new RuntimeException("컴파일 결과 YAML 파일이 생성되지 않았습니다."); - } - String yamlContent = Files.readString(yamlPath, StandardCharsets.UTF_8); - - String baseName = originalName.replaceAll("\\.py$", ""); - String yamlObjectName = "compiled/" + id + "_" + baseName + ".yaml"; - byte[] yamlBytes = yamlContent.getBytes(StandardCharsets.UTF_8); - minioClient.putObject( - PutObjectArgs.builder() - .bucket(bucketName) - .object(yamlObjectName) - .stream(new ByteArrayInputStream(yamlBytes), yamlBytes.length, -1) - .contentType("application/x-yaml") - .build() - ); - - Map result = new HashMap<>(); - result.put("yamlStoragePath", yamlObjectName); - result.put("yamlContent", yamlContent); - return result; - } finally { - try { - if (tempDir != null && Files.exists(tempDir)) { - Files.walk(tempDir).sorted((a, b) -> -a.compareTo(b)).forEach(path -> { - try { - Files.deleteIfExists(path); - } catch (Exception e) { - log.debug("temp 삭제 무시: {}", path, e); - } - }); - } - } catch (Exception e) { - log.debug("temp 정리 중 오류 무시", e); - } - } - } - - /** - * 해당 첨부파일 ID 기준으로 이미 컴파일된 YAML이 있는지 확인하고, - * 존재하면 MinIO 객체 경로를 Optional로 반환합니다. - */ - public Optional findCompiledYamlPath(Long id) { - MinioAttachmentEntity attachment = minioAttachmentRepository.findById(id) - .orElseThrow(() -> new IllegalArgumentException("첨부파일을 찾을 수 없습니다. ID=" + id)); - - String originalName = attachment.getOriginalName(); - if (originalName == null || !originalName.toLowerCase().endsWith(".py")) { - throw new IllegalArgumentException("Python(.py) 파일만 컴파일 대상입니다. 파일: " + originalName); - } - - String baseName = originalName.replaceAll("\\.py$", ""); - String yamlObjectName = "compiled/" + id + "_" + baseName + ".yaml"; - try { - minioClient.statObject( - StatObjectArgs.builder() - .bucket(bucketName) - .object(yamlObjectName) - .build() - ); - return Optional.of(yamlObjectName); - } catch (Exception e) { - log.debug("컴파일된 YAML이 존재하지 않음: id={}, objectName={}", id, yamlObjectName); - return Optional.empty(); - } - } - - /** - * 컴파일된 YAML과 원본 .py 스크립트를 하나의 ZIP으로 묶어 반환합니다. - * @param attachmentId 첨부파일 ID (원본 .py) - * @param yamlObjectName MinIO에 저장된 YAML 객체 경로 (예: compiled/5_xxx.yaml) - * @return ZIP 파일 byte[] - */ - public byte[] downloadCompiledBundle(Long attachmentId, String yamlObjectName) throws Exception { - MinioAttachmentEntity attachment = minioAttachmentRepository.findById(attachmentId) - .orElseThrow(() -> new IllegalArgumentException("첨부파일을 찾을 수 없습니다. ID=" + attachmentId)); - if (yamlObjectName == null || yamlObjectName.isBlank()) { - throw new IllegalArgumentException("yamlObjectName이 없습니다."); - } - - byte[] pyBytes = downloadFile(bucketName, attachment.getStoragePath()); - byte[] yamlBytes = downloadFile(bucketName, yamlObjectName); - - String pyFileName = attachment.getOriginalName() != null ? attachment.getOriginalName() : "script.py"; - String yamlFileName = yamlObjectName.contains("/") ? yamlObjectName.substring(yamlObjectName.lastIndexOf('/') + 1) : yamlObjectName; - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZipOutputStream zos = new ZipOutputStream(baos)) { - zos.putNextEntry(new ZipEntry(pyFileName)); - zos.write(pyBytes); - zos.closeEntry(); - zos.putNextEntry(new ZipEntry(yamlFileName)); - zos.write(yamlBytes); - zos.closeEntry(); - } - return baos.toByteArray(); - } -} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index eae0446..0498d5c 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,7 +1,7 @@ #????? ?? ?? server.port = 8080 -spring.profiles.active=local +spring.profiles.active=aws spring.datasource.url=jdbc:mariadb://192.168.10.143:3306/autoflow spring.datasource.driver-class-name=org.mariadb.jdbc.Driver