feat: 중복 기사 필터링 3단계 구현 및 최신 기사 우선 정렬 개선

SpringBoot/src/main/java/Baemin/News_Deliver/Domain/Kakao/service/KakaoNewsService.java

@@ -134,11 +134,17 @@ public List<NewsEsDocument> searchNewsByDateRange(List<String> includeKeywords,
                 );
             }
 
-            // Elasticsearch 검색 요청
+            // Elasticsearch 검색 요청 (최신 기사 우선, 관련도 보조)
             SearchRequest request = SearchRequest.of(s -> s
                     .index("news-index-nori")
                     .query(finalQuery)
                     .size(5)
+                    .sort(sort -> sort
+                            .field(f -> f
+                                    .field("published_at")
+                                    .order(SortOrder.Desc)
+                            )
+                    )
                     .sort(sort -> sort
                             .score(sc -> sc.order(SortOrder.Desc))
                     )

SpringBoot/src/main/java/Baemin/News_Deliver/Global/JWT/JwtAuthenticationFilter.java

@@ -175,7 +175,8 @@ protected boolean shouldNotFilter(HttpServletRequest request) {
         if (path.equals("/run-batch") ||
                 path.startsWith("/elasticsearch/") ||
                 path.startsWith("/monitoring/test/") ||
-                path.startsWith("/kakao/")) {
+                path.startsWith("/kakao/") ||
+                path.startsWith("/api/admin/")) {
             return true;
         }
 

SpringBoot/src/main/java/Baemin/News_Deliver/Global/News/Batch/configuration/BatchConfig.java

@@ -30,7 +30,9 @@
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 /**
  * 뉴스 배치 관련 설정 클래스
@@ -168,18 +170,33 @@ public ItemReader<NewsItemDTO> apiReader(
     /**
      * 뉴스 DTO → Entity 변환 Processor
      *
-     * <p>섹션이 존재하지 않는 뉴스는 {@code null}을 반환하여 skip 처리합니다.</p>
+     * <p>섹션이 존재하지 않거나 배치 수집 중 중복된 제목의 뉴스는 skip 처리합니다.</p>
+     * <p>제목 정규화: 대괄호/소괄호 태그 제거 후 앞 30자로 중복 판별</p>
      *
      * @return ItemProcessor
      */
     @Bean
     public ItemProcessor<NewsItemDTO, News> newsProcessor() {
-        // 받아온 DTO를 엔티티로 변환하는 과정 (하나씩 처리)
+        // 배치 실행 단위로 중복 제목 추적 (thread-safe Set)
+        Set<String> seenTitles = Collections.synchronizedSet(new java.util.HashSet<>());
 
         return dto -> {
             if (dto.getSections() == null || dto.getSections().isEmpty()) {
                 log.warn("❌ 섹션 정보 없음 → 건너뜀 (title: {})", dto.getTitle());
-                return null; // sections가 없으면 skip
+                return null;
+            }
+
+            // 제목 정규화: [태그], (태그) 제거 후 공백 제거, 앞 30자
+            String normalizedTitle = dto.getTitle()
+                    .replaceAll("\\[.*?\\]|\\(.*?\\)", "")
+                    .replaceAll("\\s+", "")
+                    .toLowerCase();
+            String titleKey = dto.getPublisher() + "::" +
+                    (normalizedTitle.length() > 30 ? normalizedTitle.substring(0, 30) : normalizedTitle);
+
+            if (!seenTitles.add(titleKey)) {
+                log.debug("⏭ 배치 내 중복 제목 skip: {}", dto.getTitle());
+                return null;
             }
 
             return News.builder()
@@ -188,7 +205,7 @@ public ItemProcessor<NewsItemDTO, News> newsProcessor() {
                     .publisher(dto.getPublisher())
                     .contentUrl(dto.getContent_url())
                     .publishedAt(dto.getPublished_at())
-                    .sections(dto.getSections().get(0)) // List → String
+                    .sections(dto.getSections().get(0))
                     .send(false)
                     .build();
         };
@@ -236,7 +253,7 @@ private void getNewsList(int page, String section, int pageSize, String dateFrom
                 .queryParam("page_size", pageSize)
                 .queryParam("date_to", dateTo)
                 .queryParam("date_from", dateFrom)
-                .queryParam("order", "published_at")
+                .queryParam("order", "-published_at")
                 .queryParam("page", page)
                 .build()
                 .toUriString();
@@ -279,7 +296,7 @@ private NewsResponseDTO getAPIResponse(int page, String section, int pageSize, S
                 .queryParam("page_size", pageSize)
                 .queryParam("date_to", dateTo)
                 .queryParam("date_from", dateFrom)
-                .queryParam("order", "published_at")
+                .queryParam("order", "-published_at")
                 .queryParam("page", page)
                 .build()
                 .toUriString();

SpringBoot/src/main/java/Baemin/News_Deliver/Global/News/Batch/listener/BatchJobCompletionListener.java

@@ -36,25 +36,75 @@ public class BatchJobCompletionListener implements JobExecutionListener {
     @Override
     public void afterJob(JobExecution jobExecution) {
         if (jobExecution.getStatus() == BatchStatus.COMPLETED) {
-            String sql = """
+
+            // 1. 완전 동일 제목+언론사 중복 제거 (가장 최신 기사 유지 - id DESC)
+            String exactDupSql = """
+                DELETE FROM news
+                WHERE id IN (
+                    SELECT id FROM (
+                        SELECT id,
+                               ROW_NUMBER() OVER (
+                                   PARTITION BY title, publisher
+                                   ORDER BY published_at DESC, id DESC
+                               ) AS rn
+                        FROM news
+                        WHERE published_at >= CURDATE() - INTERVAL 1 DAY
+                          AND published_at < CURDATE()
+                    ) t
+                    WHERE t.rn > 1
+                );
+                """;
+
+            // 2. 정규화 제목 기반 유사 중복 제거
+            //    - 대괄호/소괄호 태그 제거([속보], (종합) 등), 공백 제거 후 앞 30자 비교
+            //    - 같은 정규화 제목 그룹에서 가장 최신 기사만 유지
+            String similarDupSql = """
+                DELETE FROM news
+                WHERE id IN (
+                    SELECT id FROM (
+                        SELECT id,
+                               ROW_NUMBER() OVER (
+                                   PARTITION BY
+                                       LEFT(
+                                           REPLACE(
+                                               REGEXP_REPLACE(title, '\\\\[.*?\\\\]|\\\\(.*?\\\\)|\\\\s', ''),
+                                           ' ', ''),
+                                       30
+                                       ),
+                                       publisher
+                                   ORDER BY published_at DESC, id DESC
+                               ) AS rn
+                        FROM news
+                        WHERE published_at >= CURDATE() - INTERVAL 1 DAY
+                          AND published_at < CURDATE()
+                    ) t
+                    WHERE t.rn > 1
+                );
+                """;
+
+            // 3. 노이즈성 기사 제거 ([속보], [단독] 단독 제목, 광고성 패턴)
+            String noiseSql = """
                 DELETE FROM news
-                    WHERE (
-                        id IN (
-                            SELECT id FROM (
-                                SELECT id,
-                                       ROW_NUMBER() OVER (PARTITION BY title, publisher ORDER BY id) AS rn
-                                FROM news
-                                WHERE published_at >= CURDATE() - INTERVAL 1 DAY
-                                  AND published_at < CURDATE()
-                            ) t
-                            WHERE t.rn > 1
-                        )
-                        OR title LIKE '%[속보]%'
-                    );
+                WHERE (
+                    title REGEXP '^\\\\[(속보|단독|긴급|알림)\\\\]$'
+                    OR title LIKE '%〔%'
+                    OR title LIKE '%▶%'
+                    OR LENGTH(title) < 10
+                )
+                AND published_at >= CURDATE() - INTERVAL 1 DAY
+                AND published_at < CURDATE();
                 """;
 
-            int deleted = jdbcTemplate.update(sql);
-            log.info("✅ 중복 뉴스 삭제 완료: {}건", deleted);
+            int exactDeleted = jdbcTemplate.update(exactDupSql);
+            log.info("✅ 완전 중복 제거: {}건", exactDeleted);
+
+            int similarDeleted = jdbcTemplate.update(similarDupSql);
+            log.info("✅ 유사 제목 중복 제거: {}건", similarDeleted);
+
+            int noiseDeleted = jdbcTemplate.update(noiseSql);
+            log.info("✅ 노이즈 기사 제거: {}건", noiseDeleted);
+
+            log.info("✅ 전체 중복/노이즈 제거 완료: 총 {}건", exactDeleted + similarDeleted + noiseDeleted);
         }
     }
 }

SpringBoot/src/main/java/Baemin/News_Deliver/SecurityConfig.java

@@ -9,10 +9,13 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer;
 import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
 import org.springframework.security.config.http.SessionCreationPolicy;
 import org.springframework.security.web.SecurityFilterChain;
 import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
+import org.springframework.security.web.firewall.HttpFirewall;
+import org.springframework.security.web.firewall.StrictHttpFirewall;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.cors.CorsConfiguration;
 import org.springframework.web.cors.CorsConfigurationSource;
@@ -122,6 +125,26 @@ public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Excepti
                 .build();
     }
 
+    /**
+     * URL 경로에 한국어(비ASCII) 문자가 포함된 경우 StrictHttpFirewall이 차단하는 문제를 해결
+     * /api/hottopic/{keyword} 등 한국어 키워드를 경로 변수로 받는 엔드포인트를 위해 허용
+     */
+    @Bean
+    public HttpFirewall allowEncodedKoreanFirewall() {
+        StrictHttpFirewall firewall = new StrictHttpFirewall();
+        firewall.setAllowUrlEncodedPercent(true);
+        firewall.setAllowUrlEncodedSlash(true);
+        firewall.setAllowSemicolon(true);
+        // 비ASCII(한국어 포함) URL 인코딩된 문자 허용
+        firewall.setAllowUrlEncodedDoubleSlash(true);
+        return firewall;
+    }
+
+    @Bean
+    public WebSecurityCustomizer webSecurityCustomizer() {
+        return web -> web.httpFirewall(allowEncodedKoreanFirewall());
+    }
+
     // CORS 설정
     @Bean
     public CorsConfigurationSource corsConfigurationSource() {

docker-compose.yml

@@ -41,7 +41,8 @@ services:
       - /etc/localtime:/etc/localtime:ro
       - /etc/timezone:/etc/timezone:ro
     depends_on:
-      # mysql 의존성 제거 - RDS 사용으로 인해 불필요
+      mysql:
+        condition: service_healthy
       redis-cache:
         condition: service_started
       redis-session1:
@@ -51,33 +52,33 @@ services:
     networks:
       - backend
 
-  # MySQL 서비스 제거 - RDS 사용
-  # mysql:
-  #   image: mysql:8.0
-  #   container_name: mysql
-  #   logging:
-  #     driver: "json-file"
-  #     options:
-  #       max-size: "10m"
-  #       max-file: "3"
-  #   environment:
-  #     - TZ=Asia/Seoul
-  #     - MYSQL_ROOT_PASSWORD=${DB_PASS}
-  #     - MYSQL_DATABASE=backendDB
-  #   command: ["mysqld", "--default-time-zone=+09:00"]
-  #   ports:
-  #     - "3306:3306"
-  #   healthcheck:
-  #     test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
-  #     interval: 10s
-  #     timeout: 5s
-  #     retries: 5
-  #   volumes:
-  #     - mysql-data:/var/lib/mysql
-  #     - /etc/localtime:/etc/localtime:ro
-  #     - /etc/timezone:/etc/timezone:ro
-  #   networks:
-  #     - backend
+  mysql:
+    image: mysql:8.0
+    container_name: mysql
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+    environment:
+      - TZ=Asia/Seoul
+      - MYSQL_ROOT_PASSWORD=${DB_PASS}
+      - MYSQL_DATABASE=backendDB
+    command: ["mysqld", "--default-time-zone=+09:00"]
+    ports:
+      - "3306:3306"
+    healthcheck:
+      test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - mysql-data:/var/lib/mysql
+      - ./SpringBoot/database/init.sql:/docker-entrypoint-initdb.d/init.sql
+      - /etc/localtime:/etc/localtime:ro
+      - /etc/timezone:/etc/timezone:ro
+    networks:
+      - backend
 
   redis-cache:
     image: redis:7.2
@@ -246,7 +247,7 @@ services:
       - backend
 
 volumes:
-  # mysql-data: # RDS 사용으로 인해 불필요
+  mysql-data:
   redis-cache-data:
   redis-session1-data:
   redis-session2-data: