From abb2afcda02a32a1135201830af85565d219d362 Mon Sep 17 00:00:00 2001 From: bjkim Date: Wed, 20 May 2026 19:03:49 +0900 Subject: [PATCH] fix: force merge all remaining patched updates regardless of timestamps --- Dockerfile_server | 20 +- README.md | 270 ++++--- build.gradle.kts | 13 +- gradle/wrapper/gradle-wrapper.properties | 5 +- .../etri/autoflow/batch/BatchScheduler.java | 2 +- .../batch/KubeflowRunBatchConfig.java | 17 +- .../kr/re/etri/autoflow/common/AwsConfig.java | 27 +- .../autoflow/common/WebConfiguration.java | 21 +- .../controllers/ExternalAuthController.java | 4 +- .../ExternalDataSetController.java | 8 +- .../controllers/KubeflowRunsController.java | 32 + .../controllers/MlflowController.java | 263 ++++++- .../controllers/PipelineUploadController.java | 12 +- .../re/etri/autoflow/models/RefreshToken.java | 3 +- .../payload/response/KubeflowRunResponse.java | 2 +- .../autoflow/security/WebSecurityConfig.java | 25 +- .../autoflow/service/DataGroupService.java | 14 +- .../etri/autoflow/service/DatasetService.java | 12 +- .../autoflow/service/KubeflowRunService.java | 5 +- .../service/PipelineUploadService.java | 741 ++++++------------ .../KubeflowRunSpecification.java | 5 +- .../resources/application-local.properties | 22 +- .../resources/application-prod.properties | 3 - src/main/resources/application.properties | 66 +- src/main/resources/data.sql | 56 +- 25 files changed, 848 insertions(+), 800 deletions(-) diff --git a/Dockerfile_server b/Dockerfile_server index 0d5e2f8..b556442 100644 --- a/Dockerfile_server +++ b/Dockerfile_server @@ -1,10 +1,10 @@ -FROM openjdk:17-jdk-alpine -MAINTAINER [AutoFlow] - -RUN apk --no-cache add tzdata && cp /usr/share/zoneinfo/Asia/Seoul /etc/localtime - -RUN mkdir /server -COPY build/libs/*.jar /server/app.jar - -WORKDIR /server -ENTRYPOINT ["java", "-jar", "app.jar"] \ No newline at end of file +FROM openjdk:17-jdk-alpine +MAINTAINER [AutoFlow] + +RUN apk --no-cache add tzdata && cp /usr/share/zoneinfo/Asia/Seoul /etc/localtime + +RUN mkdir /server +ADD build/libs/autoflow-0.0.1-SNAPSHOT.jar /server/autoflow-0.0.1-SNAPSHOT.jar + +WORKDIR /server +ENTRYPOINT ["java", "-jar", "autoflow-0.0.1-SNAPSHOT.jar"] \ No newline at end of file diff --git a/README.md b/README.md index 146c9f0..b6aa2e2 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,202 @@ -# AutoFlow Server Management (autoflow-server-mgmt) +# Spring Security Refresh Token with JWT in Spring Boot example -이 프로젝트는 AutoFlow 시스템의 핵심 관리 서버로, 머신러닝 파이프라인 관리, 데이터셋 관리, 그리고 외부 시스템(Kubeflow, MLflow, OTA 등)과의 통합을 담당합니다. Spring Boot 기반의 견고한 아키텍처를 통해 데이터 기반의 워크플로우를 효율적으로 관리합니다. +Build JWT Refresh Token with Spring Security in the Spring Boot Application. You can know how to expire the JWT Token, then renew the Access Token with Refresh Token in HttpOnly Cookie. ---- +The instruction can be found at: +[Spring Security Refresh Token with JWT](https://www.bezkoder.com/spring-security-refresh-token/) -## 🚀 주요 기능 +## User Registration, User Login and Authorization process. +The diagram shows flow of how we implement User Registration, User Login and Authorization process. -### 1. 인증 및 권한 관리 (Authentication & Security) -- **JWT (JSON Web Token)** 기반 인증 지원 -- **Refresh Token**을 통한 보안성 및 사용자 편의성 강화 -- 쿠키 기반의 토큰 저장 방식 (`cuuva-jwt`, `cuuva-jwt-refresh`) -- 프로젝트 및 작업 단위별 세부 권한 제어 +![spring-security-jwt-auth-spring-boot-flow](spring-security-jwt-auth-spring-boot-flow.png) -### 2. 프로젝트 및 데이터 관리 (Project & Data Management) -- **프로젝트(Project)** 생성, 수정, 삭제 및 멤버별 권한 관리 -- **데이터 그룹(Data Group)** 및 **데이터셋(Dataset)**의 계층적 관리 -- 외부 데이터셋 연동 및 관리 기능 +And this is for Refresh Token: -### 3. ML 파이프라인 및 워크플로우 (ML Pipeline & Workflow) -- **Kubeflow** 통합: Experiments 및 Runs 관리, 파이프라인 업로드 -- **MLflow** 통합: 실험 결과 및 메트릭 추적 -- 워크플로우 생성 및 실행 관리 +![spring-security-refresh-token-jwt-spring-boot-flow](spring-security-refresh-token-jwt-spring-boot-flow.png) -### 4. 파일 관리 (File Management) -- **AWS S3** 연동을 통한 대용량 파일 저장 및 관리 -- 멀티파트(Multipart) 파일 업로드 지원 (최대 500MB) -- 동적 AWS S3 첨부 파일 관리 시스템 +## Configure Spring Datasource, JPA, App properties +Open `src/main/resources/application.properties` -### 5. 외부 시스템 연동 (External Integrations) -- **OTA (Over-The-Air)** 연동: 외부 인증 및 패키지 검색 API 연동 지원 -- **Spring Batch**를 활용한 대용량 데이터 처리 및 통계 수집 +```properties +spring.datasource.url= jdbc:mysql://localhost:3306/testdb?useSSL=false +spring.datasource.username= root +spring.datasource.password= 123456 ---- +spring.jpa.properties.hibernate.dialect= org.hibernate.dialect.MySQLDialect +spring.jpa.hibernate.ddl-auto= update -## 🛠 기술 스택 (Tech Stack) +# App Properties +bezkoder.app.jwtSecret= bezKoderSecretKey +bezkoder.app.jwtExpirationMs= 3600000 +bezkoder.app.jwtRefreshExpirationMs= 86400000 +``` -- **Language:** Java 17 -- **Framework:** Spring Boot 3.5.6 -- **Build Tool:** Gradle -- **Database:** MariaDB (JPA / Hibernate) -- **Security:** Spring Security, JWT (jjwt 0.11.5) -- **Storage:** AWS S3 -- **Batch Processing:** Spring Batch -- **API Documentation:** Springdoc OpenAPI (Swagger UI) -- **Etc:** Lombok, Jsoup, Caffeine Cache, WebFlux +## Run Spring Boot application +``` +mvn spring-boot:run +``` ---- +## Run following SQL insert statements +``` +INSERT INTO roles(name) VALUES('ROLE_USER'); +INSERT INTO roles(name) VALUES('ROLE_MODERATOR'); +INSERT INTO roles(name) VALUES('ROLE_ADMIN'); +``` -## ⚙️ 설정 방법 (Configuration) +Related Posts: +> [Spring Boot, Spring Security: JWT Authentication & Authorization example](https://www.bezkoder.com/spring-boot-security-login-jwt/) -### 사전 요구 사항 -- Java 17 이상 설치 -- MariaDB 설치 및 데이터베이스 생성 (`autoflow`) -- AWS S3 접근 권한 필요 +> [For MySQL/PostgreSQL](https://www.bezkoder.com/spring-boot-login-example-mysql/) -### 환경 설정 (`application.properties`) -`src/main/resources/application.properties` 파일에서 다음 항목들을 설정해야 합니다: +> [For MongoDB](https://www.bezkoder.com/spring-boot-mongodb-login-example/) -```properties -# 데이터베이스 설정 -spring.datasource.url=jdbc:mariadb://{DB_HOST}:3306/autoflow -spring.datasource.username={USER} -spring.datasource.password={PASSWORD} - -# JWT 보안 설정 -cuuva.app.jwtSecret={YOUR_SECRET_KEY} - -# AWS S3 설정 -cloud.aws.s3.endpoint={AWS_S3_ENDPOINT} -cloud.aws.credentials.access-key={AWS_ACCESS_KEY} -cloud.aws.credentials.secret-key={AWS_SECRET_KEY} -cloud.aws.region.static={AWS_REGION} - -# 외부 서비스 연동 -kubeflow.url={KUBEFLOW_URL} -mlflow.url={MLFLOW_URL} -``` +## More Practice: +> [Spring Boot File upload example with Multipart File](https://bezkoder.com/spring-boot-file-upload/) ---- +> [Exception handling: @RestControllerAdvice example in Spring Boot](https://bezkoder.com/spring-boot-restcontrolleradvice/) -## 🏃 실행 방법 (Getting Started) +> [Spring Boot Repository Unit Test with @DataJpaTest](https://bezkoder.com/spring-boot-unit-test-jpa-repo-datajpatest/) -### Gradle을 이용한 실행 -```bash -./gradlew bootRun -``` +> [Spring Boot Rest Controller Unit Test with @WebMvcTest](https://www.bezkoder.com/spring-boot-webmvctest/) -### 소스 빌드 및 배포 (JAR) -```bash -./gradlew build -# 빌드된 파일 위치: build/libs/autoflow-server-mgmt-0.0.1-SNAPSHOT.jar -java -jar build/libs/autoflow-server-mgmt-0.0.1-SNAPSHOT.jar -``` +> [Spring Boot Pagination & Sorting example](https://www.bezkoder.com/spring-boot-pagination-sorting-example/) -### API 문서 (Swagger) -서버 실행 후 아래 주소에서 API 명세를 확인할 수 있습니다: -- `http://localhost:8080/swagger-ui/index.html` +> Validation: [Spring Boot Validate Request Body](https://www.bezkoder.com/spring-boot-validate-request-body/) ---- +> Documentation: [Spring Boot and Swagger 3 example](https://www.bezkoder.com/spring-boot-swagger-3/) -## 📁 프로젝트 구조 (Project Structure) +> Caching: [Spring Boot Redis Cache example](https://www.bezkoder.com/spring-boot-redis-cache-example/) -``` -kr.re.etri.autoflow -├── controllers # API 컨트롤러 (Auth, Project, Data, etc.) -├── service # 비즈니스 로직 처리 -├── repository # 데이터 액세스 계층 (JPA) -├── entity # 데이터베이스 엔티티 -├── security # 시큐리티 설정 및 JWT 처리 -├── batch # Spring Batch 작업 구성 -├── payload # Request/Response DTO -├── models # 도메인 모델 -├── exception # 전역 예외 처리 -└── common # 공통 유틸리티 및 설정 -``` +Associations: +> [Spring Boot One To Many example with Spring JPA, Hibernate](https://www.bezkoder.com/jpa-one-to-many/) + +> [Spring Boot Many To Many example with Spring JPA, Hibernate](https://www.bezkoder.com/jpa-many-to-many/) + +> [JPA One To One example with Spring Boot](https://www.bezkoder.com/jpa-one-to-one/) + +Deployment: +> [Deploy Spring Boot App on AWS – Elastic Beanstalk](https://www.bezkoder.com/deploy-spring-boot-aws-eb/) + +> [Docker Compose Spring Boot and MySQL example](https://www.bezkoder.com/docker-compose-spring-boot-mysql/) + +## Fullstack Authentication + +> [Spring Boot + Vue.js JWT Authentication](https://bezkoder.com/spring-boot-vue-js-authentication-jwt-spring-security/) + +> [Spring Boot + Angular 8 JWT Authentication](https://bezkoder.com/angular-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 10 JWT Authentication](https://bezkoder.com/angular-10-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 11 JWT Authentication](https://bezkoder.com/angular-11-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 12 JWT Authentication](https://www.bezkoder.com/angular-12-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 13 JWT Authentication](https://www.bezkoder.com/angular-13-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 14 JWT Authentication](https://www.bezkoder.com/angular-14-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 15 JWT Authentication](https://www.bezkoder.com/angular-15-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 16 JWT Authentication](https://www.bezkoder.com/angular-16-spring-boot-jwt-auth/) + +> [Spring Boot + Angular 17 JWT Authentication](https://www.bezkoder.com/angular-17-spring-boot-jwt-auth/) + +> [Spring Boot + React JWT Authentication](https://bezkoder.com/spring-boot-react-jwt-auth/) + +## Fullstack CRUD App + +> [Vue.js + Spring Boot + H2 Embedded database example](https://www.bezkoder.com/spring-boot-vue-js-crud-example/) + +> [Vue.js + Spring Boot + MySQL example](https://www.bezkoder.com/spring-boot-vue-js-mysql/) + +> [Vue.js + Spring Boot + PostgreSQL example](https://www.bezkoder.com/spring-boot-vue-js-postgresql/) + +> [Angular 8 + Spring Boot + Embedded database example](https://www.bezkoder.com/angular-spring-boot-crud/) + +> [Angular 8 + Spring Boot + MySQL example](https://bezkoder.com/angular-spring-boot-crud/) + +> [Angular 8 + Spring Boot + PostgreSQL example](https://bezkoder.com/angular-spring-boot-postgresql/) + +> [Angular 10 + Spring Boot + MySQL example](https://bezkoder.com/angular-10-spring-boot-crud/) + +> [Angular 10 + Spring Boot + PostgreSQL example](https://bezkoder.com/angular-10-spring-boot-postgresql/) + +> [Angular 11 + Spring Boot + MySQL example](https://bezkoder.com/angular-11-spring-boot-crud/) + +> [Angular 11 + Spring Boot + PostgreSQL example](https://bezkoder.com/angular-11-spring-boot-postgresql/) + +> [Angular 12 + Spring Boot + Embedded database example](https://www.bezkoder.com/angular-12-spring-boot-crud/) + +> [Angular 12 + Spring Boot + MySQL example](https://www.bezkoder.com/angular-12-spring-boot-mysql/) + +> [Angular 12 + Spring Boot + PostgreSQL example](https://www.bezkoder.com/angular-12-spring-boot-postgresql/) + +> [Angular 13 + Spring Boot + H2 Embedded Database example](https://www.bezkoder.com/spring-boot-angular-13-crud/) + +> [Angular 13 + Spring Boot + MySQL example](https://www.bezkoder.com/spring-boot-angular-13-mysql/) + +> [Angular 13 + Spring Boot + PostgreSQL example](https://www.bezkoder.com/spring-boot-angular-13-postgresql/) + +> [Angular 14 + Spring Boot + H2 Embedded Database example](https://www.bezkoder.com/spring-boot-angular-14-crud/) + +> [Angular 14 + Spring Boot + MySQL example](https://www.bezkoder.com/spring-boot-angular-14-mysql/) + +> [Angular 14 + Spring Boot + PostgreSQL example](https://www.bezkoder.com/spring-boot-angular-14-postgresql/) + +> [Angular 15 + Spring Boot + H2 Embedded Database example](https://www.bezkoder.com/spring-boot-angular-15-crud/) + +> [Angular 15 + Spring Boot + MySQL example](https://www.bezkoder.com/spring-boot-angular-15-mysql/) + +> [Angular 15 + Spring Boot + PostgreSQL example](https://www.bezkoder.com/spring-boot-angular-15-postgresql/) + +> [Angular 15 + Spring Boot + MongoDB example](https://www.bezkoder.com/spring-boot-angular-15-mongodb/) + +> [Angular 16 + Spring Boot + H2 Embedded Database example](https://www.bezkoder.com/spring-boot-angular-16-crud/) + +> [Angular 16 + Spring Boot + MySQL example](https://www.bezkoder.com/spring-boot-angular-16-mysql/) + +> [Angular 16 + Spring Boot + PostgreSQL example](https://www.bezkoder.com/spring-boot-angular-16-postgresql/) + +> [Angular 16 + Spring Boot + MongoDB example](https://www.bezkoder.com/spring-boot-angular-16-mongodb/) + +> [Angular 17 + Spring Boot + H2 Embedded Database example](https://www.bezkoder.com/spring-boot-angular-17-crud/) + +> [Angular 17 + Spring Boot + MySQL example](https://www.bezkoder.com/spring-boot-angular-17-mysql/) + +> [Angular 17 + Spring Boot + PostgreSQL example](https://www.bezkoder.com/spring-boot-angular-17-postgresql/) + +> [Angular 17 + Spring Boot + MongoDB example](https://www.bezkoder.com/spring-boot-angular-17-mongodb/) + +> [React + Spring Boot + MySQL example](https://bezkoder.com/react-spring-boot-crud/) + +> [React + Spring Boot + PostgreSQL example](https://bezkoder.com/spring-boot-react-postgresql/) + +> [React + Spring Boot + MongoDB example](https://bezkoder.com/react-spring-boot-mongodb/) + +Run both Back-end & Front-end in one place: +> [Integrate Angular with Spring Boot Rest API](https://bezkoder.com/integrate-angular-spring-boot/) + +> [Integrate React.js with Spring Boot Rest API](https://bezkoder.com/integrate-reactjs-spring-boot/) + +> [Integrate Vue.js with Spring Boot Rest API](https://bezkoder.com/integrate-vue-spring-boot/) + +## More Practice: +> [Spring Boot File upload example with Multipart File](https://bezkoder.com/spring-boot-file-upload/) + +> [Exception handling: @RestControllerAdvice example in Spring Boot](https://bezkoder.com/spring-boot-restcontrolleradvice/) + +> [Spring Boot Repository Unit Test with @DataJpaTest](https://bezkoder.com/spring-boot-unit-test-jpa-repo-datajpatest/) + +> [Spring Boot Pagination & Sorting example](https://www.bezkoder.com/spring-boot-pagination-sorting-example/) + +Associations: +> [JPA/Hibernate One To Many example](https://www.bezkoder.com/jpa-one-to-many/) + +> [JPA/Hibernate Many To Many example](https://www.bezkoder.com/jpa-many-to-many/) + +> [JPA/Hibernate One To One example](https://www.bezkoder.com/jpa-one-to-one/) + +Deployment: +> [Deploy Spring Boot App on AWS – Elastic Beanstalk](https://www.bezkoder.com/deploy-spring-boot-aws-eb/) + +> [Docker Compose Spring Boot and MySQL example](https://www.bezkoder.com/docker-compose-spring-boot-mysql/) diff --git a/build.gradle.kts b/build.gradle.kts index b01463f..0d2c0f3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,6 @@ plugins { // Spring Boot - id("org.springframework.boot") version "3.5.14" + id("org.springframework.boot") version "3.5.6" // Spring 의존성 관리(BOM) id("io.spring.dependency-management") version "1.1.7" @@ -26,12 +26,16 @@ repositories { dependencies { // Spring Boot 스타터들 implementation("org.springframework.boot:spring-boot-starter-web") + implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.springframework.boot:spring-boot-starter-security") implementation("org.springframework.boot:spring-boot-starter-validation") // https://mvnrepository.com/artifact/org.springframework.batch/spring-batch-core - implementation("org.springframework.boot:spring-boot-starter-batch") + implementation("org.springframework.batch:spring-batch-core:5.2.3") +// implementation("org.springframework.boot:spring-boot-starter-batch:5.2.0") + testImplementation("org.springframework.batch:spring-batch-test:5.2.3") + // JWT implementation("io.jsonwebtoken:jjwt-api:0.11.5") @@ -59,8 +63,6 @@ dependencies { implementation("software.amazon.awssdk:s3:2.35.10") - implementation("io.kubernetes:client-java:19.0.0") - // 테스트 testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("org.springframework.security:spring-security-test") @@ -71,6 +73,9 @@ dependencies { implementation("io.minio:minio:8.5.17") + // Kubernetes API (Pod 목록 조회, phase == Running 확인) + implementation("io.kubernetes:client-java:19.0.3") + implementation("org.springframework.boot:spring-boot-starter-cache") // 캐시 지원 implementation("com.github.ben-manes.caffeine:caffeine:3.2.2")} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e2e71e4..d4081da 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ -#Fri May 08 16:07:48 KST 2026 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/kr/re/etri/autoflow/batch/BatchScheduler.java b/src/main/java/kr/re/etri/autoflow/batch/BatchScheduler.java index db225fa..d22baa2 100644 --- a/src/main/java/kr/re/etri/autoflow/batch/BatchScheduler.java +++ b/src/main/java/kr/re/etri/autoflow/batch/BatchScheduler.java @@ -17,7 +17,7 @@ public class BatchScheduler { private final JobLauncher jobLauncher; private final Job runSyncJob; // Spring Batch의 Job 타입 - @Scheduled(fixedDelay = 300000) // 30초마다 실행 + @Scheduled(fixedDelay = 10000) // 10초마다 실행 (KFP 실행 결과를 DB에 반영 → UI 목록 갱신) public void runJob() throws Exception { JobParameters params = new JobParametersBuilder() .addLong("timestamp", System.currentTimeMillis()) // 중복 실행 방지 diff --git a/src/main/java/kr/re/etri/autoflow/batch/KubeflowRunBatchConfig.java b/src/main/java/kr/re/etri/autoflow/batch/KubeflowRunBatchConfig.java index 51a5dce..d35c490 100644 --- a/src/main/java/kr/re/etri/autoflow/batch/KubeflowRunBatchConfig.java +++ b/src/main/java/kr/re/etri/autoflow/batch/KubeflowRunBatchConfig.java @@ -16,9 +16,9 @@ import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; -import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.client.reactive.ReactorClientHttpConnector; @@ -42,19 +42,20 @@ public class KubeflowRunBatchConfig { private final KubeflowRunRepository kubeflowRunRepository; private final CacheManager cacheManager; - @Value("${kubeflow.url:http://localhost:8080}") - private String kubeflowUrl; - private static final int PAGE_SIZE = 50; private static final String SORT_BY = "created_at DESC"; + @Value("${kubeflow.url:http://192.168.10.135:32473/}") + private String kubeflowBaseUrl; + @Bean public WebClient.Builder webClientBuilder() { + String baseUrl = kubeflowBaseUrl != null ? kubeflowBaseUrl.replaceAll("/+$", "") : "http://192.168.10.135:32473"; HttpClient httpClient = HttpClient.create() .responseTimeout(Duration.ofSeconds(30)); // 응답 제한 return WebClient.builder() - .baseUrl(kubeflowUrl) + .baseUrl(baseUrl) .clientConnector(new ReactorClientHttpConnector(httpClient)) .exchangeStrategies(ExchangeStrategies.builder() .codecs(configurer -> configurer.defaultCodecs() @@ -100,9 +101,9 @@ public class KubeflowRunBatchConfig { .bodyToMono(KubeflowRunResponse.class) .block(); - if (response == null || response.getRuns() == null || response.getRuns().isEmpty()) { - log.info("KubeflowRunBatch: 조회된 데이터가 없거나 응답이 비어있음"); - runs = Collections.emptyList(); // null 대신 빈 리스트 할당 + if (response == null || response.getRuns().isEmpty()) { + log.info("KubeflowRunBatch: 데이터 없음, 종료"); + runs = Collections.emptyList(); return null; } diff --git a/src/main/java/kr/re/etri/autoflow/common/AwsConfig.java b/src/main/java/kr/re/etri/autoflow/common/AwsConfig.java index 59d2a26..eb245ad 100644 --- a/src/main/java/kr/re/etri/autoflow/common/AwsConfig.java +++ b/src/main/java/kr/re/etri/autoflow/common/AwsConfig.java @@ -7,41 +7,26 @@ import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; - -import java.net.URI; @Configuration public class AwsConfig { - @Value("${storage.provider:s3}") - private String storageProvider; - - @Value("${cloud.aws.credentials.access-key:minio}") + @Value("${cloud.aws.credentials.access-key}") private String accessKey; - @Value("${cloud.aws.credentials.secret-key:minio123}") + @Value("${cloud.aws.credentials.secret-key}") private String secretKey; - @Value("${cloud.aws.region.static:ap-northeast-2}") + @Value("${cloud.aws.region.static}") private String region; - @Value("${minio.endpoint:http://localhost:9000}") - private String minioEndpoint; - @Bean public S3Client s3Client() { AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey); - S3ClientBuilder builder = S3Client.builder() + return S3Client.builder() .region(Region.of(region)) - .credentialsProvider(StaticCredentialsProvider.create(credentials)); - - if ("minio".equalsIgnoreCase(storageProvider)) { - builder.endpointOverride(URI.create(minioEndpoint)) - .forcePathStyle(true); - } - - return builder.build(); + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build(); } } diff --git a/src/main/java/kr/re/etri/autoflow/common/WebConfiguration.java b/src/main/java/kr/re/etri/autoflow/common/WebConfiguration.java index ec784b1..051bf60 100644 --- a/src/main/java/kr/re/etri/autoflow/common/WebConfiguration.java +++ b/src/main/java/kr/re/etri/autoflow/common/WebConfiguration.java @@ -1,6 +1,7 @@ package kr.re.etri.autoflow.common; import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpMethod; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @@ -9,11 +10,19 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; public class WebConfiguration implements WebMvcConfigurer { @Override - public void addInterceptors( - InterceptorRegistry registry) { + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**") + .allowedOriginPatterns("http://localhost:3000", "http://localhost:5173", "http://127.0.0.1:3000", "http://127.0.0.1:5173", "http://10.10.11.144", "http://cuuva.com:2481", "http://210.217.121.58:2481") // 허용할 Origin 지정 + .allowedMethods("GET", "POST", "PUT", "DELETE", "OPTIONS") + .allowedHeaders("*") // 필요하면 "cuuva-jwt", "Content-Type", "Authorization" 명시 가능 + .exposedHeaders("cuuva-jwt") + //.allowCredentials(true) + .maxAge(3600); + } - registry.addInterceptor( - new LoggingInterceptor()) - .addPathPatterns("/**"); + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(new LoggingInterceptor()) + .addPathPatterns("/**"); // Intercepts all requests } -} \ No newline at end of file +} diff --git a/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java b/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java index 244e423..a9ad2bc 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/ExternalAuthController.java @@ -9,7 +9,7 @@ import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.annotation.PostConstruct; import kr.re.etri.autoflow.payload.request.EdgeSWVO; -import kr.re.etri.autoflow.service.DynamicStorageAttachmentService; +import kr.re.etri.autoflow.service.DynamicMinioAttachmentService; import kr.re.etri.autoflow.service.EdgeSWUploadService; import kr.re.etri.autoflow.service.ExternalAuthService; import lombok.RequiredArgsConstructor; @@ -45,7 +45,7 @@ public class ExternalAuthController { private final ExternalAuthService externalAuthService; private final EdgeSWUploadService edgeSWUploadService; - private final DynamicStorageAttachmentService minioService; + private final DynamicMinioAttachmentService minioService; private RestTemplate restTemplate; diff --git a/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java b/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java index fd7f794..fd880e4 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/ExternalDataSetController.java @@ -7,7 +7,7 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; -import kr.re.etri.autoflow.entity.StorageAttachmentEntity; +import kr.re.etri.autoflow.entity.MinioAttachmentEntity; import kr.re.etri.autoflow.service.DatasetService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -26,7 +26,7 @@ import java.util.Map; public class ExternalDataSetController { private final DatasetService datasetService; - private final kr.re.etri.autoflow.service.StorageAttachmentService storageAttachmentService; + private final kr.re.etri.autoflow.service.MinioAttachmentService minioAttachmentService; @Operation( summary = "데이터셋 목록 조회", @@ -64,13 +64,13 @@ public class ExternalDataSetController { @RequestParam Long projectId ) { try { - StorageAttachmentEntity saved = datasetService.downloadDataset( + MinioAttachmentEntity saved = datasetService.downloadDataset( datasetName, path, refId, refType, title, description, version, regUserId, projectId ); Map response = new HashMap<>(); response.put("attachment", saved); - response.put("minioUrl", storageAttachmentService.getFileUrl(saved.getStoragePath())); + response.put("minioUrl", minioAttachmentService.getFileUrl(saved.getStoragePath())); return ResponseEntity.ok(response); } catch (Exception e) { diff --git a/src/main/java/kr/re/etri/autoflow/controllers/KubeflowRunsController.java b/src/main/java/kr/re/etri/autoflow/controllers/KubeflowRunsController.java index a79cce8..6d74e0d 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/KubeflowRunsController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/KubeflowRunsController.java @@ -7,9 +7,13 @@ import kr.re.etri.autoflow.entity.KubeflowRunEntity; import kr.re.etri.autoflow.payload.request.KubeflowRunSearchRequest; import kr.re.etri.autoflow.repository.KubeflowRunRepository; import kr.re.etri.autoflow.service.KubeflowRunService; +import kr.re.etri.autoflow.service.PipelineUploadService; import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springdoc.core.annotations.ParameterObject; import org.springframework.data.domain.Page; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -21,9 +25,11 @@ import java.util.List; @RequiredArgsConstructor public class KubeflowRunsController { + private static final Logger log = LoggerFactory.getLogger(KubeflowRunsController.class); private final KubeflowRunRepository runRepository; private final KubeflowRunService kubeflowRunService; + private final PipelineUploadService pipelineUploadService; @Operation(summary = "모든 Kubeflow Run 조회") @GetMapping @@ -48,7 +54,33 @@ public class KubeflowRunsController { public ResponseEntity> searchRuns( @ParameterObject @ModelAttribute KubeflowRunSearchRequest request) { + log.info("[KubeflowRuns] GET /api/kubeflow/runs/search 호출됨 (Run 목록 조회)"); Page page = kubeflowRunService.search(request); + int total = page.getNumberOfElements(); + String firstRunId = page.getContent().isEmpty() ? null : page.getContent().get(0).getRunId(); + log.info("[KubeflowRuns] search 응답: content 개수={}, totalElements={}, 첫 run runId={}", + total, page.getTotalElements(), firstRunId != null ? firstRunId : "(없음)"); return ResponseEntity.ok(page); } + + @Operation(summary = "Kubeflow Run 삭제 (KFP에서 삭제 성공 시에만 DB에서 제거)") + @DeleteMapping("/{runId}") + public ResponseEntity deleteRun( + @Parameter(description = "Kubeflow Run ID", example = "ad980d7f-050a-4c59-a775-94394befad40") + @PathVariable("runId") String runId) { + + return runRepository.findByRunId(runId) + .map((entity) -> { + try { + pipelineUploadService.deleteKfpRun(runId, entity.getExperimentId()); + } catch (Exception e) { + log.warn("[KubeflowRuns] KFP Run 삭제 실패, DB는 유지: runId={}, error={}", runId, e.getMessage()); + return ResponseEntity.status(HttpStatus.BAD_GATEWAY) + .body("KFP Run 삭제 실패. 목록에서 제거되지 않습니다: " + e.getMessage()); + } + runRepository.delete(entity); + return ResponseEntity.noContent().build(); + }) + .orElse(ResponseEntity.notFound().build()); + } } diff --git a/src/main/java/kr/re/etri/autoflow/controllers/MlflowController.java b/src/main/java/kr/re/etri/autoflow/controllers/MlflowController.java index 9466545..d298c04 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/MlflowController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/MlflowController.java @@ -1,30 +1,95 @@ package kr.re.etri.autoflow.controllers; +import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.util.UriComponentsBuilder; import reactor.core.publisher.Mono; +import java.net.URI; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Tag(name = "MLflow API", description = "MLflow Experiment 및 Run 조회 API") @RestController @RequestMapping("/api/mlflow") public class MlflowController { + private static final Logger log = LoggerFactory.getLogger(MlflowController.class); private final WebClient webClient; + /** MLflow 서버 루트( get-artifact 등은 /api/2.0/mlflow 가 아닌 루트에 있음 ) */ + private final WebClient artifactWebClient; + /** 정규화된 MLflow 서버 루트 (예: http://localhost:5000) */ + private final String mlflowServerRoot; + private final ObjectMapper objectMapper; - public MlflowController() { - this.webClient = WebClient.builder() - .baseUrl("http://192.168.10.135:30128/api/2.0/mlflow") - .defaultHeaders(headers -> headers.setBasicAuth("user", "WjWjIi13KEkO")) - .build(); + public MlflowController( + ObjectMapper objectMapper, + @Value("${mlflow.url:http://192.168.10.135:30128/}") String mlflowUrl, + @Value("${mlflow.user:}") String mlflowUser, + @Value("${mlflow.password:}") String mlflowPassword) { + this.objectMapper = objectMapper != null ? objectMapper : new ObjectMapper(); + String serverRoot = (mlflowUrl != null ? mlflowUrl.replaceAll("/+$", "") : "").trim(); + if (serverRoot.isBlank()) { + serverRoot = "http://localhost:5000"; + log.info("mlflow.url is blank; using default mlflowServerRoot: {}", serverRoot); + } + // 포트가 없으면 5000으로 보정 (예: http://localhost → http://localhost:5000) + if (serverRoot.matches("https?://[^:/]+$")) { + serverRoot = serverRoot + ":5000"; + log.info("mlflow.url has no port; using {} as mlflowServerRoot", serverRoot); + } + this.mlflowServerRoot = serverRoot; + + String baseUrl = mlflowServerRoot + "/api/2.0/mlflow"; + WebClient.Builder builder = WebClient.builder().baseUrl(baseUrl); + if (mlflowUser != null && !mlflowUser.isBlank() && mlflowPassword != null) { + builder.defaultHeaders(headers -> headers.setBasicAuth(mlflowUser, mlflowPassword != null ? mlflowPassword : "")); + } + this.webClient = builder.build(); + // artifactWebClient 는 baseUrl 없이 사용하고, 매 호출마다 절대 URL(mlflowServerRoot + path) 전달 + // 큰 파일(수십 MB) 다운로드를 위해 클라이언트 디코더 버퍼를 충분히 크게 설정 + WebClient.Builder artifactBuilder = WebClient.builder() + .codecs(c -> c.defaultCodecs().maxInMemorySize(128 * 1024 * 1024)); // 128MB + if (mlflowUser != null && !mlflowUser.isBlank() && mlflowPassword != null) { + artifactBuilder.defaultHeaders(headers -> headers.setBasicAuth(mlflowUser, mlflowPassword != null ? mlflowPassword : "")); + } + this.artifactWebClient = artifactBuilder.build(); + } + + @Operation( + summary = "전체 Experiment 목록 조회", + description = "MLflow의 전체 Experiment 목록을 조회합니다. experiment 이름을 일일이 등록하지 않고 동적으로 검색할 때 사용합니다.", + responses = { + @ApiResponse(responseCode = "200", description = "Experiment 목록 (experiments 배열)"), + @ApiResponse(responseCode = "500", description = "서버 오류 발생") + } + ) + @GetMapping(value = "/experiments", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity getExperiments() { + try { + String body = webClient.post() + .uri("/experiments/search") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(Map.of("max_results", 1000)) + .retrieve() + .bodyToMono(String.class) + .block(); + return ResponseEntity.ok(body != null ? body : "{\"experiments\":[]}"); + } catch (Exception e) { + return ResponseEntity.internalServerError().body(e.getMessage()); + } } @Operation( @@ -59,25 +124,140 @@ public class MlflowController { @Operation( summary = "Run 단건 조회", - description = "주어진 Run ID의 상세 정보를 조회합니다. MLflow API `/runs/get`를 호출하여 Run 정보를 반환합니다.", + description = "주어진 Run ID의 상세 정보를 조회합니다. MLflow API `/runs/get`를 호출하고, run.info.experiment_id가 있으면 Experiments 목록에서 experiment name을 조회해 run.info.experiment_name으로 보강하여 반환합니다.", responses = { - @ApiResponse(responseCode = "200", description = "Run 정보 조회 성공"), + @ApiResponse(responseCode = "200", description = "Run 정보 조회 성공 (info.experiment_name 포함)"), @ApiResponse(responseCode = "500", description = "서버 오류 발생") } ) @GetMapping(value = "/run", produces = MediaType.APPLICATION_JSON_VALUE) - public Mono> getRun( + public ResponseEntity getRun( @Parameter(description = "조회할 Run ID", required = true, example = "59e4f75b29eb4354b9e9e2ec9d93e2e3") @RequestParam String runId) { + try { + String uri = String.format("/runs/get?run_id=%s", runId); + String runBody = webClient.get() + .uri(uri) + .retrieve() + .bodyToMono(String.class) + .block(); + if (runBody == null || runBody.isBlank()) { + return ResponseEntity.ok(runBody != null ? runBody : "{}"); + } + @SuppressWarnings("unchecked") + Map runMap = objectMapper.readValue(runBody, Map.class); + Object runObj = runMap.get("run"); + Map run = runObj instanceof Map ? (Map) runObj : null; + if (run == null) { + return ResponseEntity.ok(runBody); + } + Object infoObj = run.get("info"); + Map info = infoObj instanceof Map ? (Map) infoObj : null; + Object expIdObj = info != null ? info.get("experiment_id") : null; + if (expIdObj != null && info != null) { + try { + Map expSearchResponse = webClient.post() + .uri("/experiments/search") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(Map.of("max_results", 1000)) + .retrieve() + .bodyToMono(Map.class) + .block(); + if (expSearchResponse != null && expSearchResponse.containsKey("experiments")) { + List experiments = (List) expSearchResponse.get("experiments"); + String expIdStr = String.valueOf(expIdObj); + for (Object e : experiments != null ? experiments : Collections.emptyList()) { + if (!(e instanceof Map)) continue; + Map exp = (Map) e; + Object id = exp.get("experiment_id"); + if (id != null && expIdStr.equals(String.valueOf(id))) { + Object name = exp.get("name"); + if (name != null) { + info.put("experiment_name", name.toString()); + } + break; + } + } + } + } catch (Exception e) { + log.debug("[MLflow] getRun: experiment_name 보강 실패 (무시), runId={}, error={}", runId, e.getMessage()); + } + } + return ResponseEntity.ok(objectMapper.writeValueAsString(runMap)); + } catch (Exception e) { + log.warn("[MLflow] getRun 실패: runId={}, error={}", runId, e.getMessage()); + return ResponseEntity.internalServerError().body(e.getMessage()); + } + } - String uri = String.format("/runs/get?run_id=%s", runId); + @Operation( + summary = "Kubeflow Run ID에 해당하는 MLflow Run 목록 조회", + description = "태그 kubeflow_run_id가 일치하는 Run을 전체 experiment에서 검색합니다. experiment name과 무관하게 매칭됩니다.", + responses = { + @ApiResponse(responseCode = "200", description = "Run 목록 조회 성공 (runs 배열)"), + @ApiResponse(responseCode = "500", description = "서버 오류 발생") + } + ) + @GetMapping(value = "/runs/by-kubeflow-run-id", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity getRunsByKubeflowRunId( + @Parameter(description = "Kubeflow Run ID (workflow.uid)", required = true) + @RequestParam String kubeflowRunId) { - return webClient.get() - .uri(uri) - .retrieve() - .bodyToMono(String.class) - .map(ResponseEntity::ok) - .onErrorResume(e -> Mono.just(ResponseEntity.internalServerError().body(e.getMessage()))); + if (kubeflowRunId == null || (kubeflowRunId = kubeflowRunId.trim()).isBlank()) { + log.info("[MLflow] getRunsByKubeflowRunId: kubeflowRunId 비어 있음 → runs:[]"); + return ResponseEntity.ok("{\"runs\":[]}"); + } + + log.info("[MLflow] getRunsByKubeflowRunId: kubeflowRunId={}", kubeflowRunId); + try { + Map expSearchResponse = webClient.post() + .uri("/experiments/search") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(Map.of("max_results", 1000)) + .retrieve() + .bodyToMono(Map.class) + .block(); + + List experimentIds = Collections.emptyList(); + if (expSearchResponse != null && expSearchResponse.containsKey("experiments")) { + List> experiments = (List>) expSearchResponse.get("experiments"); + if (experiments != null && !experiments.isEmpty()) { + experimentIds = experiments.stream() + .map(e -> String.valueOf(e.get("experiment_id"))) + .filter(id -> id != null && !"null".equals(id)) + .collect(Collectors.toList()); + } + } + if (experimentIds.isEmpty()) { + log.info("[MLflow] getRunsByKubeflowRunId: experiment 0개 → runs:[]"); + return ResponseEntity.ok("{\"runs\":[]}"); + } + + String escaped = kubeflowRunId.replace("'", "\\'").replace("\"", "\\\""); + String filter = "tags.kubeflow_run_id = '" + escaped + "'"; + Map runsSearchBody = Map.of( + "experiment_ids", experimentIds, + "filter", filter, + "order_by", Collections.singletonList("attribute.start_time DESC"), + "max_results", 100 + ); + + String runsResponse = webClient.post() + .uri("/runs/search") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(runsSearchBody) + .retrieve() + .bodyToMono(String.class) + .block(); + + String body = runsResponse != null ? runsResponse : "{\"runs\":[]}"; + int runCount = body.contains("\"run_id\"") ? body.split("\"run_id\"").length - 1 : 0; + log.info("[MLflow] getRunsByKubeflowRunId: kubeflowRunId={} → run 수 약 {}개", kubeflowRunId, runCount); + return ResponseEntity.ok(body); + } catch (Exception e) { + log.warn("[MLflow] getRunsByKubeflowRunId 실패: kubeflowRunId={}, error={}", kubeflowRunId, e.getMessage()); + return ResponseEntity.internalServerError().body(e.getMessage()); + } } @Operation( @@ -149,4 +329,57 @@ public class MlflowController { .map(ResponseEntity::ok) .onErrorResume(e -> Mono.just(ResponseEntity.internalServerError().body(e.getMessage()))); } + + @Operation( + summary = "Artifact 파일 다운로드 (MLflow 프록시)", + description = """ + MLflow의 get-artifact API를 프록시하여 Run의 artifact 파일을 다운로드합니다. + MinIO 직접 경로(experiment_id/run_id/artifacts/...)와 달리 MLflow가 저장한 실제 경로를 사용하므로 NoSuchKey 오류를 방지할 수 있습니다. + """, + responses = { + @ApiResponse(responseCode = "200", description = "파일 스트림"), + @ApiResponse(responseCode = "404", description = "Run 또는 artifact 없음"), + @ApiResponse(responseCode = "500", description = "서버 오류") + } + ) + @GetMapping(value = "/artifacts/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) + public Mono> downloadArtifact( + @Parameter(description = "MLflow Run ID", required = true) @RequestParam(name = "run_id") String runId, + @Parameter(description = "artifact 상대 경로 (예: outputs/sample_model.txt)", required = true) @RequestParam String path) { + URI uri = UriComponentsBuilder.fromHttpUrl(mlflowServerRoot) + .path("/get-artifact") + .queryParam("run_id", runId) + .queryParam("path", path) + .build() + .toUri(); + return artifactWebClient.get() + .uri(uri) + .exchangeToMono(response -> { + if (!response.statusCode().is2xxSuccessful()) { + return response.bodyToMono(String.class) + .defaultIfEmpty(response.statusCode().toString()) + .map(msg -> ResponseEntity.status(response.statusCode()).body(msg.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + return response.bodyToMono(byte[].class) + .map(body -> { + var headers = response.headers().asHttpHeaders(); + ResponseEntity.BodyBuilder builder = ResponseEntity.ok(); + if (headers.getFirst(org.springframework.http.HttpHeaders.CONTENT_DISPOSITION) != null) { + builder = builder.header( + org.springframework.http.HttpHeaders.CONTENT_DISPOSITION, + headers.getFirst(org.springframework.http.HttpHeaders.CONTENT_DISPOSITION) + ); + } + if (headers.getContentType() != null) { + builder = builder.contentType(headers.getContentType()); + } + return builder.body(body); + }); + }) + .onErrorResume(e -> { + log.warn("MLflow artifact download failed: run_id={}, path={}", runId, path, e); + return Mono.just(ResponseEntity.internalServerError() + .body(("Artifact download failed: " + e.getMessage()).getBytes(java.nio.charset.StandardCharsets.UTF_8))); + }); + } } diff --git a/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java b/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java index 44e205f..1fdef87 100644 --- a/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java +++ b/src/main/java/kr/re/etri/autoflow/controllers/PipelineUploadController.java @@ -6,10 +6,10 @@ import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; -import kr.re.etri.autoflow.entity.StorageAttachmentEntity; +import kr.re.etri.autoflow.entity.MinioAttachmentEntity; import kr.re.etri.autoflow.entity.WorkflowEntity; import kr.re.etri.autoflow.payload.request.CreateRunRequest; -import kr.re.etri.autoflow.service.StorageAttachmentService; +import kr.re.etri.autoflow.service.MinioAttachmentService; import kr.re.etri.autoflow.service.PipelineUploadService; import kr.re.etri.autoflow.service.WorkFlowService; import lombok.RequiredArgsConstructor; @@ -36,7 +36,7 @@ public class PipelineUploadController { private final PipelineUploadService pipelineUploadService; private final WorkFlowService workFlowService; - private final StorageAttachmentService storageAttachmentService; + private final MinioAttachmentService minioAttachmentService; @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResponseEntity> uploadPipeline( @@ -73,7 +73,7 @@ public class PipelineUploadController { workFlowService.save(workflow); // 2. MinIO 업로드 - StorageAttachmentEntity attachment = storageAttachmentService.uploadFile( + MinioAttachmentEntity attachment = minioAttachmentService.uploadFile( file, "workflows/" + projectId, workflow.getId(), @@ -85,14 +85,14 @@ public class PipelineUploadController { projectId ); - String minioUrl = storageAttachmentService.getFileUrl(attachment.getStoragePath()); + String minioUrl = minioAttachmentService.getFileUrl(attachment.getStoragePath()); // 3. 최종 응답 Map response = new HashMap<>(); response.put("pipeline", result); response.put("workflow", workflow); response.put("attachment", attachment); - response.put("storageUrl", minioUrl); + response.put("minioUrl", minioUrl); return ResponseEntity.ok(response); diff --git a/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java b/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java index a78137d..68e237f 100644 --- a/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java +++ b/src/main/java/kr/re/etri/autoflow/models/RefreshToken.java @@ -16,8 +16,7 @@ import org.hibernate.annotations.Comment; public class RefreshToken { @Id - @SequenceGenerator(name = "refreshtoken_seq", sequenceName = "tb_refreshtoken_seq", allocationSize = 1) - @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "refreshtoken_seq") + @GeneratedValue(strategy = GenerationType.AUTO) private long id; @OneToOne diff --git a/src/main/java/kr/re/etri/autoflow/payload/response/KubeflowRunResponse.java b/src/main/java/kr/re/etri/autoflow/payload/response/KubeflowRunResponse.java index 609b4b7..a86dd05 100644 --- a/src/main/java/kr/re/etri/autoflow/payload/response/KubeflowRunResponse.java +++ b/src/main/java/kr/re/etri/autoflow/payload/response/KubeflowRunResponse.java @@ -7,6 +7,6 @@ import java.util.List; @Data public class KubeflowRunResponse { - private List runs = new java.util.ArrayList<>(); + private List runs; private int totalSize; } \ No newline at end of file diff --git a/src/main/java/kr/re/etri/autoflow/security/WebSecurityConfig.java b/src/main/java/kr/re/etri/autoflow/security/WebSecurityConfig.java index 495e7e2..6f79fd6 100644 --- a/src/main/java/kr/re/etri/autoflow/security/WebSecurityConfig.java +++ b/src/main/java/kr/re/etri/autoflow/security/WebSecurityConfig.java @@ -17,9 +17,6 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; -import org.springframework.web.cors.CorsConfiguration; -import org.springframework.web.cors.CorsConfigurationSource; -import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import kr.re.etri.autoflow.security.jwt.AuthEntryPointJwt; import kr.re.etri.autoflow.security.jwt.AuthTokenFilter; @@ -106,37 +103,19 @@ public class WebSecurityConfig { // extends WebSecurityConfigurerAdapter { // return http.build(); // } + // 임시 설정 @Bean public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { http.csrf(AbstractHttpConfigurer::disable) - .cors(cors -> cors.configurationSource(corsConfigurationSource())) // CORS 설정 추가 .exceptionHandling(exception -> exception.authenticationEntryPoint(unauthorizedHandler)) .sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) .authorizeHttpRequests(auth -> - auth.requestMatchers("/actuator/**").permitAll() - .anyRequest().permitAll() + auth.anyRequest().permitAll() // 모든 요청 허용 ); - http.authenticationProvider(authenticationProvider()); http.addFilterBefore(authenticationJwtTokenFilter(), UsernamePasswordAuthenticationFilter.class); return http.build(); } - - @Bean - public CorsConfigurationSource corsConfigurationSource() { - CorsConfiguration configuration = new CorsConfiguration(); - configuration.setAllowCredentials(true); - configuration.addAllowedOrigin("http://localhost:3000"); - configuration.addAllowedOrigin("http://10.10.11.144"); - configuration.addAllowedOrigin("http://cuuva.com:2481"); - configuration.addAllowedOrigin("http://210.217.121.58:2481"); - configuration.addAllowedOrigin("http://172.28.248.98:30819"); - configuration.addAllowedHeader("*"); - configuration.addAllowedMethod("*"); - UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); - source.registerCorsConfiguration("/**", configuration); - return source; - } } diff --git a/src/main/java/kr/re/etri/autoflow/service/DataGroupService.java b/src/main/java/kr/re/etri/autoflow/service/DataGroupService.java index 3a0e2c7..ae2c53d 100644 --- a/src/main/java/kr/re/etri/autoflow/service/DataGroupService.java +++ b/src/main/java/kr/re/etri/autoflow/service/DataGroupService.java @@ -3,12 +3,12 @@ package kr.re.etri.autoflow.service; import io.minio.MinioClient; import io.minio.RemoveObjectArgs; import kr.re.etri.autoflow.entity.DataGroupEntity; -import kr.re.etri.autoflow.entity.StorageAttachmentEntity; +import kr.re.etri.autoflow.entity.MinioAttachmentEntity; import kr.re.etri.autoflow.payload.request.ProjectBaseAndRefTypeRequest; import kr.re.etri.autoflow.payload.request.ProjectBaseSearchRequest; import kr.re.etri.autoflow.payload.request.ProjectRequest; import kr.re.etri.autoflow.repository.DataGroupRepository; -import kr.re.etri.autoflow.repository.StorageAttachmentRepository; +import kr.re.etri.autoflow.repository.MinioAttachmentRepository; import kr.re.etri.autoflow.specification.DataGroupSpecification; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -36,7 +36,7 @@ public class DataGroupService { private final DataGroupRepository dataGroupRepository; - private final StorageAttachmentRepository storageAttachmentRepository; + private final MinioAttachmentRepository minioAttachmentRepository; private final DataGroupSpecification dataGroupSpecification; @@ -133,11 +133,11 @@ public class DataGroupService { } // 2. refId 기준으로 MinIO 첨부파일 조회 - List attachments = - storageAttachmentRepository.findAllByRefId(dataGroupId); + List attachments = + minioAttachmentRepository.findAllByRefId(dataGroupId); // 3. MinIO에서 파일 삭제 - for (StorageAttachmentEntity attachment : attachments) { + for (MinioAttachmentEntity attachment : attachments) { try { minioClient.removeObject( RemoveObjectArgs.builder() @@ -146,7 +146,7 @@ public class DataGroupService { .build() ); // DB에서도 첨부파일 삭제 - storageAttachmentRepository.delete(attachment); + minioAttachmentRepository.delete(attachment); } catch (Exception e) { log.error("MinIO 파일 삭제 실패: {}", attachment.getStoragePath(), e); } diff --git a/src/main/java/kr/re/etri/autoflow/service/DatasetService.java b/src/main/java/kr/re/etri/autoflow/service/DatasetService.java index f98bc3f..af1a52b 100644 --- a/src/main/java/kr/re/etri/autoflow/service/DatasetService.java +++ b/src/main/java/kr/re/etri/autoflow/service/DatasetService.java @@ -2,8 +2,8 @@ package kr.re.etri.autoflow.service; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import kr.re.etri.autoflow.entity.StorageAttachmentEntity; -import kr.re.etri.autoflow.repository.StorageAttachmentRepository; +import kr.re.etri.autoflow.entity.MinioAttachmentEntity; +import kr.re.etri.autoflow.repository.MinioAttachmentRepository; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -33,7 +33,7 @@ public class DatasetService { private final RestTemplate restTemplate; - private final StorageAttachmentRepository storageAttachmentRepository; + private final MinioAttachmentRepository minioAttachmentRepository; private static final String BASE_URL = "http://52.14.11.43:18010"; @@ -111,7 +111,7 @@ public class DatasetService { .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) .build(); - public StorageAttachmentEntity downloadDataset( + public MinioAttachmentEntity downloadDataset( String datasetName, String path, Long refId, @@ -195,7 +195,7 @@ public class DatasetService { latch.await(); // DB 저장 시 size 컬럼 필수 - StorageAttachmentEntity attachment = StorageAttachmentEntity.builder() + MinioAttachmentEntity attachment = MinioAttachmentEntity.builder() .refId(refId) .refType(refType) .originalName(datasetName + ".zip") @@ -210,7 +210,7 @@ public class DatasetService { .size(totalBytes[0]) .build(); - return storageAttachmentRepository.save(attachment); + return minioAttachmentRepository.save(attachment); } catch (Exception e) { log.error("외부 API 다운로드 및 MinIO 업로드 실패", e); diff --git a/src/main/java/kr/re/etri/autoflow/service/KubeflowRunService.java b/src/main/java/kr/re/etri/autoflow/service/KubeflowRunService.java index dd95df8..1951fcf 100644 --- a/src/main/java/kr/re/etri/autoflow/service/KubeflowRunService.java +++ b/src/main/java/kr/re/etri/autoflow/service/KubeflowRunService.java @@ -26,7 +26,8 @@ public class KubeflowRunService { @Transactional(readOnly = true) public Page search(KubeflowRunSearchRequest request) { - int pageIndex = request.getPage() > 0 ? request.getPage() - 1 : 0; + // 프론트가 0-based page 전달 (0=첫페이지, 1=두번째페이지) + int pageIndex = Math.max(0, request.getPage()); Pageable pageable = PageRequest.of( pageIndex, @@ -39,7 +40,7 @@ public class KubeflowRunService { Specification spec = runSpecification.searchByConditions( - request.getExperimentId(), // experimentId는 필수 + request.getExperimentId(), request.getSearchType(), request.getKeyword(), startDate, diff --git a/src/main/java/kr/re/etri/autoflow/service/PipelineUploadService.java b/src/main/java/kr/re/etri/autoflow/service/PipelineUploadService.java index e7ed144..32ba489 100644 --- a/src/main/java/kr/re/etri/autoflow/service/PipelineUploadService.java +++ b/src/main/java/kr/re/etri/autoflow/service/PipelineUploadService.java @@ -4,16 +4,22 @@ import kr.re.etri.autoflow.payload.request.CreateRunRequest; import kr.re.etri.autoflow.payload.request.RunCreatedEvent; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; -import org.springframework.core.io.InputStreamResource; -import org.springframework.http.*; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestTemplate; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.reactive.function.BodyInserters; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientResponseException; import org.springframework.web.util.UriComponentsBuilder; @@ -21,10 +27,13 @@ import org.springframework.web.util.UriUtils; import reactor.core.publisher.Mono; import java.io.IOException; -import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; @Service @@ -34,629 +43,323 @@ public class PipelineUploadService { private final RestTemplate restTemplate; - private final WebClient webClient; - @Value("${kubeflow.url}") - private String kubeflowBaseUrl; + private String kubeflowBaseUrl; // 예: http://192.168.10.135:32473/ + + private final WebClient webClient; @Autowired private ApplicationEventPublisher eventPublisher; + /** * Pipeline 업로드 */ - public Map uploadPipeline( - MultipartFile file, - String name, - String displayName, - String description, - String namespace - ) { - + public Map uploadPipeline(MultipartFile file, + String name, + String displayName, + String description, + String namespace) { try { - - log.info(""" - ===== Pipeline Upload Start ===== - filename={} - name={} - displayName={} - description={} - namespace={} - """, - file.getOriginalFilename(), - name, - displayName, - description, - namespace - ); - - MultiValueMap body = - new LinkedMultiValueMap<>(); - - body.add( - "uploadfile", - new MultipartInputStreamFileResource( - file.getInputStream(), - file.getOriginalFilename() - ) - ); + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("uploadfile", new MultipartInputStreamFileResource(file.getInputStream(), file.getOriginalFilename())); HttpHeaders headers = new HttpHeaders(); - headers.setContentType( - MediaType.MULTIPART_FORM_DATA - ); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); - HttpEntity> - requestEntity = - new HttpEntity<>(body, headers); + HttpEntity> requestEntity = new HttpEntity<>(body, headers); - URI uri = UriComponentsBuilder - .fromHttpUrl( - normalizeBaseUrl( - kubeflowBaseUrl - ) - ) - .path("/apis/v2beta1/pipelines/upload") - .queryParamIfPresent( - "name", - optional(name) - ) - .queryParamIfPresent( - "display_name", - optional(displayName) - ) - .queryParamIfPresent( - "description", - optional(description) - ) - .queryParamIfPresent( - "namespace", - optional(namespace) - ) - .build(true) - .toUri(); + UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(kubeflowBaseUrl.replaceAll("/+$", "") + "/apis/v2beta1/pipelines/upload"); - log.info("Pipeline Upload URI={}", uri); - - ResponseEntity response = - restTemplate.postForEntity( - uri, - requestEntity, - Map.class - ); + if (name != null && !name.isBlank()) builder.queryParam("name", name); + if (displayName != null && !displayName.isBlank()) builder.queryParam("display_name", displayName); + if (description != null && !description.isBlank()) builder.queryParam("description", description); + if (namespace != null && !namespace.isBlank()) builder.queryParam("namespace", namespace); + ResponseEntity response = restTemplate.postForEntity(builder.toUriString(), requestEntity, Map.class); return response.getBody(); } catch (IOException e) { - - log.error( - "Pipeline upload failed", - e - ); - - throw new RuntimeException( - "Pipeline upload failed", - e - ); + throw new RuntimeException("Pipeline upload failed", e); } } /** * Run 생성 + * runRequest에는 display_name, pipeline_version_reference, runtime_config 등이 포함되어야 함. + * KFP v2beta1는 runtime_config 필수, display_name 필수. + * parameters는 파이프라인 스펙에 정의된 입력만 전달하며, mlflow_experiment_name은 파이프라인에 정의돼 있을 때만 포함한다. */ - public Map createRun( - CreateRunRequest runRequest - ) { - - Set allowedParamNames = - getPipelineRootParameterNames( - runRequest - ); - - Map body = - buildKfpRunRequestBody( - runRequest, - allowedParamNames - ); - - String uri = - normalizeBaseUrl(kubeflowBaseUrl) - + "/apis/v2beta1/runs"; - - log.info("Create Run URI={}", uri); + public Map createRun(CreateRunRequest runRequest) { + Set allowedParamNames = getPipelineRootParameterNames(runRequest); + Map body = buildKfpRunRequestBody(runRequest, allowedParamNames); + log.debug("[KFP] CreateRun request body: {}", body); Map result = webClient.post() - .uri(uri) - .contentType( - MediaType.APPLICATION_JSON - ) + .uri(kubeflowBaseUrl.replaceAll("/+$", "") + "/apis/v2beta1/runs") + .contentType(MediaType.APPLICATION_JSON) .bodyValue(body) .retrieve() - .onStatus( - status -> - status.is4xxClientError() - || status.is5xxServerError(), - response -> - response.bodyToMono(String.class) - .map(msg -> - new RuntimeException( - "KFP CreateRun failed: " - + response.statusCode() - + " " - + msg - ) - ) - ) + .onStatus(status -> status.is4xxClientError() || status.is5xxServerError(), + resp -> resp.bodyToMono(String.class) + .doOnNext(msg -> log.warn("[KFP] CreateRun error {}: {}", resp.statusCode(), msg)) + .map(msg -> new RuntimeException("KFP CreateRun failed: " + resp.statusCode() + " " + msg))) .bodyToMono(Map.class) .block(); - if (result != null && - result.get("run_id") != null) { - - String runId = - (String) result.get("run_id"); - - CompletableFuture.runAsync(() -> - eventPublisher.publishEvent( - new RunCreatedEvent(runId) - ) - ); + // 이벤트 발행만 비동기로 처리 + if (result != null && result.get("run_id") != null) { + String runId = (String) result.get("run_id"); + CompletableFuture.runAsync(() -> eventPublisher.publishEvent(new RunCreatedEvent(runId))); } return result; } /** - * Pipeline Version Spec 조회 + * KFP pipeline version 스펙을 조회하여 root 입력 파라미터 이름 목록을 반환한다. + * pipeline_version_reference에 pipeline_id, pipeline_version_id가 있을 때만 조회하며, 실패 시 null 반환. */ @SuppressWarnings("unchecked") - private Set getPipelineRootParameterNames( - CreateRunRequest runRequest - ) { - - CreateRunRequest.PipelineVersionReference ref = - runRequest != null - ? runRequest.getPipeline_version_reference() - : null; - - if (ref == null - || ref.getPipeline_id() == null - || ref.getPipeline_version_id() == null) { - + private Set getPipelineRootParameterNames(CreateRunRequest runRequest) { + CreateRunRequest.PipelineVersionReference ref = runRequest != null ? runRequest.getPipeline_version_reference() : null; + if (ref == null || ref.getPipeline_id() == null || ref.getPipeline_id().isBlank() + || ref.getPipeline_version_id() == null || ref.getPipeline_version_id().isBlank()) { return null; } - - String url = - normalizeBaseUrl(kubeflowBaseUrl) - + "/apis/v2beta1/pipelines/" - + ref.getPipeline_id() - + "/versions/" - + ref.getPipeline_version_id(); - + String url = kubeflowBaseUrl.replaceAll("/+$", "") + "/apis/v2beta1/pipelines/" + ref.getPipeline_id() + "/versions/" + ref.getPipeline_version_id(); try { - - Map version = - webClient.get() - .uri(url) - .retrieve() - .bodyToMono(Map.class) - .block(); - - if (version == null) { - return null; - } - - Map spec = - (Map) - version.get("pipeline_spec"); - - if (spec == null) { - return null; - } - - Map root = - (Map) - spec.get("root"); - - if (root == null) { - root = - (Map) - spec.get("Root"); - } - - if (root == null) { - return null; - } - - Map inputDefinitions = - (Map) - root.get("inputDefinitions"); - - if (inputDefinitions == null) { - inputDefinitions = - (Map) - root.get("input_definitions"); - } - - if (inputDefinitions == null) { - return null; - } - - Object parameters = - inputDefinitions.get("parameters"); - - if (!(parameters instanceof Map map)) { - return Collections.emptySet(); - } - + Map version = webClient.get() + .uri(url) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + .bodyToMono(Map.class) + .block(); + if (version == null) return null; + Object spec = version.get("pipeline_spec"); + if (!(spec instanceof Map)) return null; + Map specMap = (Map) spec; + Object root = specMap.get("root"); + if (root == null) root = specMap.get("Root"); + if (!(root instanceof Map)) return null; + Map rootMap = (Map) root; + Object inputDef = rootMap.get("inputDefinitions"); + if (inputDef == null) inputDef = rootMap.get("input_definitions"); + if (!(inputDef instanceof Map)) return null; + Map inputDefMap = (Map) inputDef; + Object params = inputDefMap.get("parameters"); + if (params == null) return Collections.emptySet(); Set names = new HashSet<>(); - - for (Object key : map.keySet()) { - names.add(key.toString()); + if (params instanceof Map) { + for (Object key : ((Map) params).keySet()) { + if (key != null) names.add(key.toString()); + } + return names; } - - return names; - + if (params instanceof Iterable) { + for (Object item : (Iterable) params) { + if (item instanceof Map) { + Object name = ((Map) item).get("name"); + if (name != null) names.add(name.toString()); + } + } + return names; + } + return Collections.emptySet(); } catch (Exception e) { - - log.warn( - "Pipeline version spec 조회 실패", - e - ); - + log.warn("[KFP] Pipeline version spec 조회 실패, parameters 필터 없이 전달 (mlflow_experiment_name 제외): {}", e.getMessage()); return null; } } /** - * Run Request Body 생성 + * KFP Run 생성 API 요청 body 구성. + * v2beta1 Run: experiment_id, display_name(필수), runtime_config(필수), pipeline_version_reference 등. + * parameters는 파이프라인 스펙에 정의된 입력만 포함한다. allowedParamNames가 null이면 스펙 조회 실패로, mlflow_experiment_name은 넣지 않고 그 외 파라미터만 전달한다. */ - private Map buildKfpRunRequestBody( - CreateRunRequest runRequest, - Set allowedParamNames - ) { - - Map body = - new HashMap<>(); - - if (runRequest.getExperiment_id() != null && - !runRequest.getExperiment_id().isBlank()) { - - body.put( - "experiment_id", - runRequest.getExperiment_id() - ); + private Map buildKfpRunRequestBody(CreateRunRequest runRequest, Set allowedParamNames) { + Map body = new HashMap<>(); + if (runRequest.getExperiment_id() != null && !runRequest.getExperiment_id().isBlank()) { + body.put("experiment_id", runRequest.getExperiment_id()); } - - body.put( - "display_name", - runRequest.getDisplay_name() != null - && !runRequest.getDisplay_name().isBlank() - ? runRequest.getDisplay_name() - : "Run" - ); - - if (runRequest.getDescription() != null && - !runRequest.getDescription().isBlank()) { - - body.put( - "description", - runRequest.getDescription() - ); + // display_name 필수 (KFP v2beta1) + body.put("display_name", runRequest.getDisplay_name() != null && !runRequest.getDisplay_name().isBlank() + ? runRequest.getDisplay_name() + : "Run"); + if (runRequest.getDescription() != null && !runRequest.getDescription().isBlank()) { + body.put("description", runRequest.getDescription()); } - - if (runRequest.getPipeline_version_reference() - != null) { - - body.put( - "pipeline_version_reference", - runRequest.getPipeline_version_reference() - ); + if (runRequest.getPipeline_version_reference() != null) { + body.put("pipeline_version_reference", runRequest.getPipeline_version_reference()); } - - Map runtimeConfig = - new HashMap<>(); - - if (runRequest.getRuntime_config() != null - && runRequest.getRuntime_config() - .getParameters() != null) { - - Map filtered = - new HashMap<>(); - - for (Map.Entry entry : - runRequest.getRuntime_config() - .getParameters() - .entrySet()) { - - String key = entry.getKey(); - - if (allowedParamNames != null && - !allowedParamNames.contains(key)) { - - continue; + if (runRequest.getService_account() != null && !runRequest.getService_account().isBlank()) { + body.put("service_account", runRequest.getService_account()); + } + // runtime_config 필수 (KFP v2beta1). 파이프라인에 정의된 파라미터만 전달. 정의되지 않으면 mlflow_experiment_name은 넣지 않음. + Map runtimeConfig = new HashMap<>(); + if (runRequest.getRuntime_config() != null && runRequest.getRuntime_config().getParameters() != null + && !runRequest.getRuntime_config().getParameters().isEmpty()) { + Map params = runRequest.getRuntime_config().getParameters(); + Map kfpParams = new HashMap<>(); + for (Map.Entry e : params.entrySet()) { + String key = e.getKey(); + if (key == null) continue; + if (allowedParamNames != null) { + if (!allowedParamNames.contains(key)) continue; + } else { + // 스펙 조회 실패 시: mlflow_* 파라미터는 파이프라인에 없을 수 있으므로 제외 + if (key.startsWith("mlflow_")) continue; } - - filtered.put( - key, - entry.getValue() - ); + kfpParams.put(key, e.getValue()); } - - if (!filtered.isEmpty()) { - - runtimeConfig.put( - "parameters", - filtered - ); + if (!kfpParams.isEmpty()) { + runtimeConfig.put("parameters", kfpParams); } } - - body.put( - "runtime_config", - runtimeConfig - ); - + body.put("runtime_config", runtimeConfig); return body; } /** * Experiments 조회 */ - public Map listExperiments( - String namespace, - int pageSize, - String pageToken - ) { + public Map listExperiments(String namespace, int pageSize, String pageToken) { + try { + UriComponentsBuilder builder = UriComponentsBuilder + .fromHttpUrl(kubeflowBaseUrl.replaceAll("/+$", "") + "/apis/v2beta1/experiments"); - URI uri = UriComponentsBuilder - .fromHttpUrl( - normalizeBaseUrl( - kubeflowBaseUrl - ) - ) - .path("/apis/v2beta1/experiments") - .queryParamIfPresent( - "namespace", - optional(namespace) - ) - .queryParamIfPresent( - "page_token", - optional(pageToken) - ) - .queryParam( - "page_size", - pageSize - ) - .build(true) - .toUri(); + if (namespace != null && !namespace.isBlank()) { + builder.queryParam("namespace", namespace); + } + if (pageSize > 0) { + builder.queryParam("page_size", pageSize); + } + if (pageToken != null && !pageToken.isBlank()) { + builder.queryParam("page_token", pageToken); + } - return webClient.get() - .uri(uri) - .retrieve() - .bodyToMono(Map.class) - .block(); + return webClient.get() + .uri(builder.toUriString()) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + .bodyToMono(Map.class) + .block(); + + } catch (Exception e) { + throw new RuntimeException("Kubeflow Experiments 조회 실패", e); + } } /** * Experiment 단건 조회 */ - public Map getExperimentById( - String experimentId - ) { + public Map getExperimentById(String experimentId) { + try { + String url = kubeflowBaseUrl.replaceAll("/+$", "") + "/apis/v2beta1/experiments/" + experimentId; - String uri = - normalizeBaseUrl(kubeflowBaseUrl) - + "/apis/v2beta1/experiments/" - + experimentId; + return webClient.get() + .uri(url) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + .bodyToMono(Map.class) + .block(); - return webClient.get() - .uri(uri) - .retrieve() - .bodyToMono(Map.class) - .block(); + } catch (Exception e) { + throw new RuntimeException("Kubeflow experiment 조회 실패: " + experimentId, e); + } } /** - * Run 단건 조회 + * KFP Run 단건 조회 (v2beta1 GET /apis/v2beta1/runs/{run_id}). + * run_details.task_details[].pod_name 으로 UI 로그와 동일한 Pod 지정 가능. */ - public Map getKfpRunById( - String runId - ) { - - if (runId == null || - runId.isBlank()) { - + @SuppressWarnings("unchecked") + public Map getKfpRunById(String runId) { + if (runId == null || runId.isBlank()) { return null; } - - String uri = - normalizeBaseUrl(kubeflowBaseUrl) - + "/apis/v2beta1/runs/" - + runId.trim(); - + String url = kubeflowBaseUrl.replaceAll("/+$", "") + "/apis/v2beta1/runs/" + runId.trim(); try { - return webClient.get() - .uri(uri) + .uri(url) .accept(MediaType.APPLICATION_JSON) .retrieve() .bodyToMono(Map.class) .block(); - } catch (WebClientResponseException e) { - if (e.getStatusCode().value() == 404) { - - log.debug( - "Run not found. runId={}", - runId - ); - - return null; + log.debug("[KFP] Run not found: {}", runId); + } else { + log.warn("[KFP] GetRun {}: {}", runId, e.getMessage()); } - - throw e; + return null; + } catch (Exception e) { + log.debug("[KFP] GetRun failed {}: {}", runId, e.getMessage()); + return null; } } /** - * Node Log 조회 + * KFP ml-pipeline이 클러스터에서 읽는 Pod 로그 (KFP UI와 동일한 백엔드 경로). + * {@code GET /apis/v1beta1/runs/{run_id}/nodes/{node_id}/log} + *

v2 Run ID도 일부 배포에서 동작합니다. 404/비어 있으면 null.

*/ - public String getV1beta1RunNodeLog( - String runId, - String nodeId - ) { - - if (runId == null || - nodeId == null) { - + public String getV1beta1RunNodeLog(String runId, String nodeId) { + if (runId == null || runId.isBlank() || nodeId == null || nodeId.isBlank()) { return null; } - - String uri = - normalizeBaseUrl(kubeflowBaseUrl) - + "/apis/v1beta1/runs/" - + UriUtils.encodePathSegment( - runId, - StandardCharsets.UTF_8 - ) - + "/nodes/" - + UriUtils.encodePathSegment( - nodeId, - StandardCharsets.UTF_8 - ) - + "/log"; - + String base = kubeflowBaseUrl.replaceAll("/+$", ""); + String encRun = UriUtils.encodePathSegment(runId.trim(), StandardCharsets.UTF_8); + String encNode = UriUtils.encodePathSegment(nodeId.trim(), StandardCharsets.UTF_8); + String url = base + "/apis/v1beta1/runs/" + encRun + "/nodes/" + encNode + "/log"; try { - return webClient.get() - .uri(uri) - .accept(MediaType.TEXT_PLAIN) + .uri(url) + .accept(MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON) .retrieve() .bodyToMono(String.class) .timeout(Duration.ofSeconds(120)) .block(); - + } catch (WebClientResponseException e) { + int code = e.getStatusCode().value(); + if (code == 404 || code == 400) { + log.debug("[KFP] v1beta1 node log {} node={}: {}", runId, nodeId, code); + } else { + log.debug("[KFP] v1beta1 node log {} node={}: {}", runId, nodeId, e.getMessage()); + } + return null; } catch (Exception e) { - - log.debug( - "Node log 조회 실패", - e - ); - + log.debug("[KFP] v1beta1 node log failed runId={} node={}: {}", runId, nodeId, e.getMessage()); return null; } } /** - * Run 삭제 - */ - public void deleteKfpRun( - String runId, - String experimentId - ) { - - String uri = - normalizeBaseUrl(kubeflowBaseUrl) - + "/apis/v2beta1/runs/" - + runId; - - if (experimentId != null && - !experimentId.isBlank()) { - - uri += "?experiment_id=" - + experimentId; - } - - webClient.delete() - .uri(uri) - .retrieve() - .onStatus( - status -> - status.is4xxClientError() - || status.is5xxServerError(), - response -> - response.bodyToMono(String.class) - .map(msg -> - new RuntimeException( - "KFP DeleteRun failed: " - + msg - ) - ) - ) - .toBodilessEntity() - .block(); - } - - /** - * Base URL normalize - */ - private String normalizeBaseUrl( - String baseUrl - ) { - - if (baseUrl == null || - baseUrl.isBlank()) { - - throw new IllegalArgumentException( - "kubeflow.url is empty" - ); - } - - baseUrl = baseUrl.trim(); - - if (baseUrl.endsWith("/")) { - - baseUrl = - baseUrl.substring( - 0, - baseUrl.length() - 1 - ); - } - - return baseUrl; - } - - private Optional optional( - String value - ) { - - if (value == null || - value.isBlank()) { - - return Optional.empty(); - } - - return Optional.of(value); - } - - /** - * Multipart Resource + * KFP Run 삭제 (v2beta1 DELETE /apis/v2beta1/runs/{run_id}). + * 성공(2xx) 또는 이미 없음(404)일 때만 정상 반환. 그 외 4xx/5xx는 예외. */ - private static class MultipartInputStreamFileResource - extends InputStreamResource { - - private final String filename; - - public MultipartInputStreamFileResource( - java.io.InputStream inputStream, - String filename - ) { - - super(inputStream); - - this.filename = filename; + public void deleteKfpRun(String runId, String experimentId) { + String base = kubeflowBaseUrl.replaceAll("/+$", ""); + String url = base + "/apis/v2beta1/runs/" + runId; + if (experimentId != null && !experimentId.isBlank()) { + url = url + "?experiment_id=" + experimentId; } - - @Override - public String getFilename() { - return filename; - } - - @Override - public long contentLength() { - return -1; + try { + webClient.delete() + .uri(url) + .retrieve() + .onStatus(status -> status.value() == 404, resp -> Mono.empty()) + .onStatus(status -> status.is4xxClientError() || status.is5xxServerError(), + resp -> resp.bodyToMono(String.class) + .doOnNext(msg -> log.warn("[KFP] DeleteRun error {}: {}", resp.statusCode(), msg)) + .map(msg -> new RuntimeException("KFP DeleteRun failed: " + resp.statusCode() + " " + msg))) + .toBodilessEntity() + .block(); + } catch (Exception e) { + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } + throw new RuntimeException("KFP Run 삭제 실패: " + runId, e); } } -} \ No newline at end of file +} diff --git a/src/main/java/kr/re/etri/autoflow/specification/KubeflowRunSpecification.java b/src/main/java/kr/re/etri/autoflow/specification/KubeflowRunSpecification.java index 6de8c9a..753cae8 100644 --- a/src/main/java/kr/re/etri/autoflow/specification/KubeflowRunSpecification.java +++ b/src/main/java/kr/re/etri/autoflow/specification/KubeflowRunSpecification.java @@ -121,7 +121,10 @@ public class KubeflowRunSpecification { return (root, query, cb) -> { Predicate predicate = cb.conjunction(); - predicate = cb.and(predicate, cb.equal(root.get("experimentId"), experimentId)); + // experimentId가 있을 때만 필터 (비어 있으면 전체 목록) + if (experimentId != null && !experimentId.isBlank()) { + predicate = cb.and(predicate, cb.equal(root.get("experimentId"), experimentId)); + } if (keyword != null && !keyword.isEmpty()) { if (searchType == null || searchType.isEmpty() || diff --git a/src/main/resources/application-local.properties b/src/main/resources/application-local.properties index 944258c..7bb1d4a 100644 --- a/src/main/resources/application-local.properties +++ b/src/main/resources/application-local.properties @@ -4,18 +4,14 @@ springdoc.swagger-ui.url=/v3/api-docs springdoc.swagger-ui.doc-expansion=none springdoc.swagger-ui.disable-swagger-default-url=true -# Local MariaDB -spring.datasource.url=jdbc:mariadb://${RDS_HOSTNAME:localhost}:3306/autoflow -spring.datasource.username=${RDS_USERNAME:cuuva} -spring.datasource.password=${RDS_PASSWORD:cuuva} +spring.jpa.hibernate.ddl-auto=none +spring.sql.init.mode=never -# Local MinIO -storage.provider=minio -minio.endpoint=${MINIO_ENDPOINT:http://localhost:9000} -minio.access-key=minio -minio.secret-key=minio123 -minio.bucket=mlpipeline - -spring.jpa.hibernate.ddl-auto=update -spring.sql.init.mode=always +# 스크립트 컴파일: Windows에서 서버 실행 시 Python 실행 파일 (python.org 설치 후 pip install kfp) +# kfp.compile.python-command=python +# MinIO type2(MLflow): wsl-port-forwards.sh 로 9001->9000 포워드 시 로컬에서 백엔드가 localhost:9001 로 접근 +minio.type2.endpoint=http://localhost:9001 +minio.type2.bucket=mlflow +minio.type2.access-key=minio-mlflow +minio.type2.secret-key=minio-mlflow-12345 diff --git a/src/main/resources/application-prod.properties b/src/main/resources/application-prod.properties index fb59cf1..f9e724e 100644 --- a/src/main/resources/application-prod.properties +++ b/src/main/resources/application-prod.properties @@ -7,6 +7,3 @@ springdoc.swagger-ui.doc-expansion=none springdoc.swagger-ui.disable-swagger-default-url=true spring.jpa.hibernate.ddl-auto=none spring.sql.init.mode=never - -# ALB / Forwarded Headers -server.forward-headers-strategy=native diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index bf7062d..eae0446 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,7 +1,7 @@ #????? ?? ?? server.port = 8080 -spring.profiles.active=aws +spring.profiles.active=local spring.datasource.url=jdbc:mariadb://192.168.10.143:3306/autoflow spring.datasource.driver-class-name=org.mariadb.jdbc.Driver @@ -50,18 +50,58 @@ spring.servlet.multipart.max-request-size=500MB springdoc.swagger-ui.tags-sorter=alpha -# Storage Provider (minio, s3, or filesystem) -storage.provider=minio - -# Local FileSystem ?? -storage.local.base-path=/app/storage -storage.local.default-bucket=mlpipeline - -# MinIO ?? +# MinIO (기본 단일 MinIO - MinIOConfig 등에서 사용) minio.endpoint=http://192.168.10.135:31795 minio.access-key=minio minio.secret-key=minio123 minio.bucket=mlpipeline +# Pod에서 접근할 MinIO 주소 (비우면 프론트에서 입력). YAML 생성 시 저장된 정보로 반영됨 +minio.endpoint.pod=http://minio-service.kubeflow.svc.cluster.local:9000 + +# MinIO type1(Kubeflow/파이프라인), type2(MLflow) - DynamicMinioAttachmentService +minio.type1.endpoint=http://192.168.10.135:31795 +minio.type1.bucket=mlpipeline +minio.type1.access-key=minio +minio.type1.secret-key=minio123 +minio.type2.endpoint=http://192.168.10.135:31000 +minio.type2.bucket=mlflow +# MLflow 아티팩트용 MinIO 계정 (YOLO 파이프라인과 동일: minio-mlflow / minio-mlflow-12345) +minio.type2.access-key=minio-mlflow +minio.type2.secret-key=minio-mlflow-12345 + +# KFP 스크립트 컴파일 (py → YAML). 서버에 Python3 + kfp 필요: pip install kfp +# Linux/WSL: python3, Windows: python +kfp.compile.python-command=python3 + +# Kubeflow +kubeflow.url=http://192.168.10.135:32473/ + +# Admin: Kubernetes Pod 상태/로그 (관리자 페이지에서 Pod Running 여부 및 로그 조회) +# kubectl get pods -n kubeflow --show-labels 로 확인한 라벨 기준 +# Argo Server (포트포워드 예: kubectl -n argo port-forward svc/argo-server 2746:2746) +# 비우면 Argo 프록시 비활성. 설정 시 로그는 여기를 최우선 시도(실행 중 K8s·종료 후 MinIO 아카이브). +argo.server.url= +# argo.auth token 또는 서비스 계정 토큰 (비우면 무인증 — 클러스터 내부 직통 시) +argo.server.token= +argo.server.container=main + +admin.k8s.enabled=true +# Argo Server URL이 있으면 로그 API를 KFP/K8s보다 먼저 시도 +admin.logs.prefer-argo-server=true +# KFP ml-pipeline v1beta1 노드 로그 (Argo 실패 시) +admin.k8s.prefer-kfp-api-for-logs=true +admin.k8s.namespace=kubeflow +# 파이프라인 Run Pod 네임스페이스(쉼표 구분, 앞에서부터 시도). 멀티유저 KFP는 보통 프로필 NS (예: kubeflow-user-example-com). 비우면 admin.k8s.namespace 만 사용. +# admin.k8s.pipeline-pod-namespaces=kubeflow-user-example-com,kubeflow +admin.k8s.pipeline-pod-namespaces= +# KFP: ml-pipeline, cache-server, metadata-*, workflow-controller 등 (application-crd-id 공통) +admin.k8s.kfp.label-selector=application-crd-id=kubeflow-pipelines +# MLflow: app=mlflow-server +admin.k8s.mlflow.label-selector=app=mlflow-server +# MinIO: minio-datasets, minio-mlflow (app=minio 없음) +admin.k8s.minio.label-selector=app in (minio-datasets,minio-mlflow) +# Run별 Pod 조회 시 라벨 (라벨키=runId). 기본 pipeline/runid. Tekton 등은 tekton.dev/pipelineRun 등이 될 수 있음 → kubectl get pods -n kubeflow --show-labels 로 확인 +# admin.k8s.run-pod-label=pipeline/runid # MLflow mlflow.url=http://192.168.10.135:30128/ @@ -81,10 +121,4 @@ external.auth.sw-search-url=https://a659120d3e2ff43ff94087b29396fd96-1057696791. cloud.aws.region.static=ap-northeast-2 cloud.aws.credentials.access-key=AKIA2UC3EPERDDR4UOWN -cloud.aws.credentials.secret-key=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 -cloud.aws.s3.bucket=mlpipeline - -# Spring Actuator Configuration -management.endpoints.web.exposure.include=health -management.endpoint.health.show-details=always -management.health.defaults.enabled=true +cloud.aws.credentials.secret-key=Ps7ShmtcemhhTmZi+aUCpSpfZxjqFGyy51qgDSGD \ No newline at end of file diff --git a/src/main/resources/data.sql b/src/main/resources/data.sql index 6eae1d8..cd608e9 100644 --- a/src/main/resources/data.sql +++ b/src/main/resources/data.sql @@ -1,47 +1,26 @@ --- tb_project ---INSERT INTO `tb_project` VALUES ---('2025-09-22','2025-09-22',1,'2025-09-22 14:28:51.507010','2025-09-22 14:28:51.507010','N','cuuva,admin','cuuva,admin','PRJ1758518911644','배터리 학습입니다.','AI 배터리 학습','cuuva,admin','cuuva,admin'); - +-- tb_project (컬럼명 명시: Hibernate 스키마 순서) +INSERT INTO `tb_project` (id, del_yn, mod_date, mod_user_id, mod_user_nm, prj_cd, prj_desc, prj_end_dt, prj_nm, prj_start_dt, reg_date, reg_user_id, reg_user_nm) VALUES +(1, 'N', '2025-09-22 14:28:51.507010', 'cuuva,admin', 'cuuva,admin', 'PRJ1758518911644', '배터리 학습입니다.', '2025-09-22', 'AI 배터리 학습', '2025-09-22', '2025-09-22 14:28:51.507010', 'cuuva,admin', 'cuuva,admin'); -- tb_role -INSERT IGNORE INTO `tb_role` VALUES +INSERT INTO `tb_role` (id, name) VALUES (1,'ROLE_USER'), (2,'ROLE_MODERATOR'), (3,'ROLE_ADMIN'); -- tb_user -INSERT IGNORE INTO tb_user ( - id, - username, - email, - password -) VALUES -( - 5, - 'cuuva', - 'cuuva@naver.com', - '$2a$10$UhWIoxGlxa7u9gks3m498u9tPGcGO2sh5PTeAD6319TJ9M67ZZqmO' -), -( - 6, - 'admin', - 'admin@naver.com', - '$2a$10$zukuiEA7Ce1ygOeJxZilhOi29jQnsreIswyJQ3Z.lysmKFiQhTXeS' -), -( - 7, - 'user', - 'user@naver.com', - '$2a$10$jkRSrScnLK.Qiy/AmapKmOVauP4tff.tIMnAzEd1mMoTvRCZXpU4u' -); +INSERT INTO `tb_user` (id, username, email, password) VALUES +(5,'cuuva','cuuva@naver.com','$2a$10$UhWIoxGlxa7u9gks3m498u9tPGcGO2sh5PTeAD6319TJ9M67ZZqmO'), +(6,'admin','admin@naver.com','$2a$10$zukuiEA7Ce1ygOeJxZilhOi29jQnsreIswyJQ3Z.lysmKFiQhTXeS'), +(7,'user','user@naver.com','$2a$10$jkRSrScnLK.Qiy/AmapKmOVauP4tff.tIMnAzEd1mMoTvRCZXpU4u'); -- tb_user_project_map -INSERT IGNORE INTO `tb_user_project_map` VALUES +INSERT INTO `tb_user_project_map` (id, project_id, user_id) VALUES (1,1,6), (2,1,5); -- tb_user_project_permission -INSERT IGNORE INTO `tb_user_project_permission` VALUES +INSERT INTO `tb_user_project_permission` (user_project_id, permissions) VALUES (2,'READ'), (1,'READ'), (2,'CREATE'), @@ -51,12 +30,17 @@ INSERT IGNORE INTO `tb_user_project_permission` VALUES (2,'DELETE'), (1,'DELETE'); --- tb_user_roles -INSERT IGNORE INTO `tb_user_roles` VALUES -(1,7), -(2,6), -(3,5); +-- tb_user_roles (user_id, role_id) - user 7=USER, 6=MODERATOR, 5=ADMIN +INSERT INTO `tb_user_roles` (user_id, role_id) VALUES +(7,1), +(6,2), +(5,3); + +-- Spring Batch 5.2 (MariaDB 10.3+): 시퀀스 생성 (최초 1회만 성공, 재실행 시 이미 있으면 무시하려면 continue-on-error=true) +CREATE SEQUENCE IF NOT EXISTS BATCH_JOB_SEQ START WITH 1 MINVALUE 1 MAXVALUE 9223372036854775806 INCREMENT BY 1 NOCACHE NOCYCLE ENGINE=InnoDB; +CREATE SEQUENCE IF NOT EXISTS BATCH_JOB_EXECUTION_SEQ START WITH 1 MINVALUE 1 MAXVALUE 9223372036854775806 INCREMENT BY 1 NOCACHE NOCYCLE ENGINE=InnoDB; +CREATE SEQUENCE IF NOT EXISTS BATCH_STEP_EXECUTION_SEQ START WITH 1 MINVALUE 1 MAXVALUE 9223372036854775806 INCREMENT BY 1 NOCACHE NOCYCLE ENGINE=InnoDB; -- 테이블 생성 (이미 존재하면 생성 안 함) CREATE TABLE IF NOT EXISTS BATCH_JOB_INSTANCE (