diff --git a/src/main/java/mServer/crawler/FilmeSuchen.java b/src/main/java/mServer/crawler/FilmeSuchen.java index a43c57021..bdb7d4294 100644 --- a/src/main/java/mServer/crawler/FilmeSuchen.java +++ b/src/main/java/mServer/crawler/FilmeSuchen.java @@ -30,7 +30,7 @@ import mServer.crawler.sender.arte.*; import mServer.crawler.sender.dreisat.DreiSatCrawler; import mServer.crawler.sender.dw.DwCrawler; -import mServer.crawler.sender.kika.KikaApiCrawler; +import mServer.crawler.sender.kika.KikaCrawler; import mServer.crawler.sender.orfon.OrfOnCrawler; import mServer.crawler.sender.phoenix.PhoenixCrawler; import mServer.crawler.sender.sr.SrCrawler; @@ -100,7 +100,7 @@ public FilmeSuchen() { mediathekListe.add(new DwCrawler(this, 0)); } if (crawlerList.contains("KIKA")) { - mediathekListe.add(new KikaApiCrawler(this, 0)); + mediathekListe.add(new KikaCrawler(this, 0)); } if (crawlerList.contains("3SAT")) { mediathekListe.add(new DreiSatCrawler(this, 0)); diff --git a/src/main/java/mServer/crawler/sender/kika/KikaApiConstants.java b/src/main/java/mServer/crawler/sender/kika/KikaApiConstants.java deleted file mode 100644 index c121df309..000000000 --- a/src/main/java/mServer/crawler/sender/kika/KikaApiConstants.java +++ /dev/null @@ -1,11 +0,0 @@ -package mServer.crawler.sender.kika; - -public final class KikaApiConstants { - - public static final String BASE_URL = "https://www.kika.de/api/v1/kikaplayer/kikaapp/"; - public static final String ALL_VIDEOS = BASE_URL + "api/videos?limit=400&orderBy=date&orderDirection=DESC"; - public static final String WEBSITE = "https://www.kika.de"; - public static final String FILM = BASE_URL + "/api/videos/%s/player-assets"; - - private KikaApiConstants() {} -} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaApiCrawler.java b/src/main/java/mServer/crawler/sender/kika/KikaApiCrawler.java deleted file mode 100644 index 2832c5fb8..000000000 --- a/src/main/java/mServer/crawler/sender/kika/KikaApiCrawler.java +++ /dev/null @@ -1,68 +0,0 @@ -package mServer.crawler.sender.kika; - - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import de.mediathekview.mlib.Const; -import de.mediathekview.mlib.daten.DatenFilm; -import de.mediathekview.mlib.tool.Log; -import mServer.crawler.CrawlerTool; -import mServer.crawler.FilmeSuchen; -import mServer.crawler.sender.MediathekCrawler; -import mServer.crawler.sender.base.JsoupConnection; -import mServer.crawler.sender.kika.tasks.KikaApiFilmTask; -import mServer.crawler.sender.kika.tasks.KikaApiTopicTask; -import mServer.crawler.sender.base.TopicUrlDTO; - -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.RecursiveTask; - -public class KikaApiCrawler extends MediathekCrawler { - private static final Logger LOG = LogManager.getLogger(KikaApiCrawler.class); - - JsoupConnection jsoupConnection; - - public KikaApiCrawler(FilmeSuchen ssearch, int startPrio) { - super(ssearch, Const.KIKA, 0, 1, startPrio); - jsoupConnection = new JsoupConnection(); - } - - - @Override - protected RecursiveTask> createCrawlerTask() { - int maxPages = 3; - - if (CrawlerTool.loadShort()) { - maxPages = 3; - } else if (CrawlerTool.loadLong()) { - maxPages = 10; - } else if (CrawlerTool.loadMax()) { - maxPages = 999; - } else if (CrawlerTool.loadLongMax()) { - maxPages = 999; - } - - try { - final ConcurrentLinkedQueue overivew = new ConcurrentLinkedQueue<>(); - overivew.add(new TopicUrlDTO("starting point", KikaApiConstants.ALL_VIDEOS)); - // - KikaApiTopicTask allvideosTask = new KikaApiTopicTask(this, overivew, 0, maxPages); - final ConcurrentLinkedQueue allVideos = new ConcurrentLinkedQueue<>(); - allVideos.addAll(forkJoinPool.invoke(allvideosTask)); - Log.sysLog("KIKA: Anzahl sendungsfolgen urls: " + allVideos.size()); - meldungAddMax(allVideos.size()); - return new KikaApiFilmTask( - this, new ConcurrentLinkedQueue<>(allVideos)); - } catch (final Exception ex) { - LOG.fatal("Exception in KIKA crawler.", ex); - Log.errorLog(324978332, ex); - } - - - return null; - - - } -} \ No newline at end of file diff --git a/src/main/java/mServer/crawler/sender/kika/KikaApiFilmDto.java b/src/main/java/mServer/crawler/sender/kika/KikaApiFilmDto.java deleted file mode 100644 index 935c897be..000000000 --- a/src/main/java/mServer/crawler/sender/kika/KikaApiFilmDto.java +++ /dev/null @@ -1,94 +0,0 @@ -package mServer.crawler.sender.kika; - -import java.util.Optional; - -import mServer.crawler.sender.base.CrawlerUrlDTO; - -public class KikaApiFilmDto extends CrawlerUrlDTO { - private Optional errorMesssage = Optional.empty(); - private Optional errorCode = Optional.empty(); - private Optional topic = Optional.empty(); - private Optional title = Optional.empty(); - private Optional id = Optional.empty(); - private Optional description = Optional.empty(); - private Optional date = Optional.empty(); - private Optional duration = Optional.empty(); - private Optional geoProtection = Optional.empty(); - private Optional website = Optional.empty(); - // - private Optional expirationDate = Optional.empty(); - private Optional appearDate = Optional.empty(); - - public KikaApiFilmDto(String aUrl, Optional topic, Optional title, Optional id, - Optional description, Optional date, Optional duration, - Optional geoProtection, Optional expirationDate, Optional appearDate, - Optional website) { - super(aUrl); - this.topic = topic; - this.title = title; - this.id = id; - this.expirationDate = expirationDate; - this.appearDate = appearDate; - this.description = description; - this.date = date; - this.duration = duration; - this.geoProtection = geoProtection; - this.website = website; - } - - public void setTopic(Optional aTopic) { - topic = aTopic; - } - public Optional getTopic() { - return topic; - } - - public Optional getTitle() { - return title; - } - - public Optional getId() { - return id; - } - - public Optional getDescription() { - return description; - } - - public Optional getDate() { - return date; - } - - public Optional getDuration() { - return duration; - } - - public Optional getGeoProtection() { - return geoProtection; - } - - public Optional getExpirationDate() { - return expirationDate; - } - - public Optional getAppearDate() { - return appearDate; - } - - public Optional getWebsite() { - return website; - } - - public void setError(Optional aErrorCode, Optional aErrorMesssage) { - errorCode = aErrorCode; - errorMesssage = aErrorMesssage; - } - - public Optional getErrorMesssage() { - return errorMesssage; - } - - public Optional getErrorCode() { - return errorCode; - } -} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaApiTopicDto.java b/src/main/java/mServer/crawler/sender/kika/KikaApiTopicDto.java deleted file mode 100644 index 7e11939bc..000000000 --- a/src/main/java/mServer/crawler/sender/kika/KikaApiTopicDto.java +++ /dev/null @@ -1,45 +0,0 @@ -package mServer.crawler.sender.kika; - -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; - -import mServer.crawler.sender.base.TopicUrlDTO; - - -public class KikaApiTopicDto { - private Optional errorMesssage = Optional.empty(); - private Optional errorCode = Optional.empty(); - private Optional nextUrl = Optional.empty(); - private Set elements = new HashSet<>(); - - public void add(KikaApiFilmDto aKikaApiFilmDto) { - elements.add(aKikaApiFilmDto); - } - - public Set getElements() { - return elements; - } - - public Optional getNextPage() { - return nextUrl; - } - - public void setNextPage(TopicUrlDTO aCrawlerUrlDTO) { - nextUrl = Optional.of(aCrawlerUrlDTO); - } - - public void setError(Optional aErrorCode, Optional aErrorMesssage) { - errorCode = aErrorCode; - errorMesssage = aErrorMesssage; - } - - public Optional getErrorMesssage() { - return errorMesssage; - } - - public Optional getErrorCode() { - return errorCode; - } - -} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaApiVideoInfoDto.java b/src/main/java/mServer/crawler/sender/kika/KikaApiVideoInfoDto.java deleted file mode 100644 index 8f65501d1..000000000 --- a/src/main/java/mServer/crawler/sender/kika/KikaApiVideoInfoDto.java +++ /dev/null @@ -1,53 +0,0 @@ -package mServer.crawler.sender.kika; - -import java.util.EnumMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -public class KikaApiVideoInfoDto { - // - private Optional errorMesssage = Optional.empty(); - private Optional errorCode = Optional.empty(); - private Map videoUrls = new EnumMap<>(Resolution.class); - private boolean hasSubtitle = false; - private Set subtitles = new HashSet<>(); - - public void addUrl(Resolution aResolution, String aFilmUrl) { - videoUrls.put(aResolution, aFilmUrl); - } - - public Map getVideoUrls() { - return videoUrls; - } - - public void addSubtitle(String aUrl ) { - subtitles.add(aUrl); - } - - public Set getSubtitle() { - return subtitles; - } - - public void setError(Optional aErrorCode, Optional aErrorMesssage) { - errorCode = aErrorCode; - errorMesssage = aErrorMesssage; - } - - public Optional getErrorMesssage() { - return errorMesssage; - } - - public Optional getErrorCode() { - return errorCode; - } - - public boolean hasSubtitle() { - return hasSubtitle; - } - - public void setSubtitle(boolean hasSub) { - hasSubtitle = hasSub; - } -} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaAssetDto.java b/src/main/java/mServer/crawler/sender/kika/KikaAssetDto.java new file mode 100644 index 000000000..2b581b8fd --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/KikaAssetDto.java @@ -0,0 +1,136 @@ +package mServer.crawler.sender.kika; + +import mServer.crawler.sender.base.Qualities; + +import java.util.Objects; +import java.util.Optional; + +public class KikaAssetDto { + Optional videoSubtitle = Optional.empty(); + Optional webvttUrl = Optional.empty(); + Optional profileName = Optional.empty(); // "Video 2014 | MP4 Web L | 16:9 | 960x540" + Optional fileName = Optional.empty(); // "FCMS-5858cc7f-3288-4101-b92e-f496f2063a37-31e0be270130_58.mp4" + Optional fileSize = Optional.empty(); // 30301969 + Optional mediaType = Optional.empty(); // null + Optional frameWidth = Optional.empty(); // 960 + Optional frameHeight = Optional.empty(); // 540 + Optional bitrateVideo = Optional.empty(); // 1800000 + Optional bitrateAudio = Optional.empty(); // 192000 + Optional type = Optional.empty(); // "progressive" + Optional url = Optional.empty(); // "https://kika-progressive.ard-mcdn.de/kika_de-prod/online/mp4dyn/5/FCMS-5858cc7f-3288-4101-b92e-f496f2063a37-31e0be270130_58.mp4" + Optional resolution = Optional.empty(); + + public KikaAssetDto(Optional profileName, Optional fileName, Optional fileSize, + Optional mediaType, Optional frameWidth, Optional frameHeight, + Optional bitrateVideo, Optional bitrateAudio, Optional type, Optional url, + Optional videoSubtitle, Optional webvttUrl) { + super(); + this.profileName = profileName; + this.fileName = fileName; + this.fileSize = fileSize; + this.mediaType = mediaType; + this.frameWidth = frameWidth; + this.frameHeight = frameHeight; + this.bitrateVideo = bitrateVideo; + this.bitrateAudio = bitrateAudio; + this.type = type; + this.videoSubtitle = videoSubtitle; + this.webvttUrl = webvttUrl; + this.url = url; + if (frameWidth.isPresent()) { + this.resolution = Optional.of(getResolutionFromWidth(frameWidth.get())); + } + } + // ALTE Logik...es gibt kein FULL HD in der Quelle + private static Qualities getResolutionFromWidth(final int width) { + if (width >= 1280) { + return Qualities.HD; + } else if (width > 640) { + return Qualities.NORMAL; + } else { + return Qualities.SMALL; + } + } + + public Optional getProfileName() { + return profileName; + } + public Optional getFileName() { + return fileName; + } + public Optional getFileSize() { + return fileSize; + } + public Optional getMediaType() { + return mediaType; + } + public Optional getFrameWidth() { + return frameWidth; + } + public Optional getFrameHeight() { + return frameHeight; + } + public Optional getBitrateVideo() { + return bitrateVideo; + } + public Optional getBitrateAudio() { + return bitrateAudio; + } + public Optional getType() { + return type; + } + public Optional getUrl() { + return url; + } + public Optional getResolution() { return resolution; } + public Optional getVideoSubtitle() { + return videoSubtitle; + } + public Optional getWebvttUrl() { + return webvttUrl; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof KikaAssetDto)) return false; + KikaAssetDto that = (KikaAssetDto) o; + return Objects.equals(profileName, that.profileName) + && Objects.equals(fileName, that.fileName) + && Objects.equals(fileSize, that.fileSize) + && Objects.equals(mediaType, that.mediaType) + && Objects.equals(frameWidth, that.frameWidth) + && Objects.equals(frameHeight, that.frameHeight) + && Objects.equals(bitrateVideo, that.bitrateVideo) + && Objects.equals(bitrateAudio, that.bitrateAudio) + && Objects.equals(type, that.type) + && Objects.equals(url, that.url) + && Objects.equals(videoSubtitle, that.videoSubtitle) + && Objects.equals(webvttUrl, that.webvttUrl); + } + + @Override + public int hashCode() { + return Objects.hash(profileName, fileName, fileSize, mediaType, frameWidth, + frameHeight, bitrateVideo, bitrateAudio, type, url, videoSubtitle, webvttUrl); + } + + @Override + public String toString() { + return "KikaAssetDto{" + + "profileName=" + profileName + + ", fileName=" + fileName + + ", fileSize=" + fileSize + + ", mediaType=" + mediaType + + ", frameWidth=" + frameWidth + + ", frameHeight=" + frameHeight + + ", bitrateVideo=" + bitrateVideo + + ", bitrateAudio=" + bitrateAudio + + ", type=" + type + + ", url=" + url + + ", videoSubtitle=" + videoSubtitle + + ", webvttUrl=" + webvttUrl + + '}'; + } + +} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaConstants.java b/src/main/java/mServer/crawler/sender/kika/KikaConstants.java new file mode 100644 index 000000000..8c1dc940d --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/KikaConstants.java @@ -0,0 +1,11 @@ +package mServer.crawler.sender.kika; + +public final class KikaConstants { + // + public static final String HOST = "https://www.kika.de"; + // + public static final String URL_LETTER_PAGE = HOST + "/ackley/v1/search/broadcast-search102?q=%s"; + // + private KikaConstants() {} + // +} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaCrawler.java b/src/main/java/mServer/crawler/sender/kika/KikaCrawler.java new file mode 100644 index 000000000..684a4a265 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/KikaCrawler.java @@ -0,0 +1,107 @@ +package mServer.crawler.sender.kika; + +import de.mediathekview.mlib.Const; +import de.mediathekview.mlib.daten.DatenFilm; +import de.mediathekview.mlib.tool.Log; +import mServer.crawler.CrawlerTool; +import mServer.crawler.FilmeSuchen; +import mServer.crawler.sender.MediathekCrawler; +import mServer.crawler.sender.base.CrawlerUrlDTO; +import mServer.crawler.sender.kika.tasks.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.RecursiveTask; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class KikaCrawler extends MediathekCrawler { + private static final Logger LOG = LogManager.getLogger(KikaCrawler.class); + + public KikaCrawler(FilmeSuchen ssearch, int startPrio) { + super(ssearch, Const.KIKA, 0, 1, startPrio); + } + + @Override + protected RecursiveTask> createCrawlerTask() { + int maxPages = 0; + + if (CrawlerTool.loadShort()) { + maxPages = 1; + } else if (CrawlerTool.loadLong()) { + maxPages = 10; + } else if (CrawlerTool.loadMax()) { + maxPages = 999; + } else if (CrawlerTool.loadLongMax()) { + maxPages = 999; + } + + try { + final ConcurrentLinkedQueue root = new ConcurrentLinkedQueue<>(); + for (char c = 'A'; c <= 'Z'; c++) { + root.add(new CrawlerUrlDTO(String.format(KikaConstants.URL_LETTER_PAGE, c))); + } + root.add(new CrawlerUrlDTO(String.format(KikaConstants.URL_LETTER_PAGE, "..."))); + + // we get channel and broadcastSeries (brand) from letters + final KikaLetterPageTask aKikaLetterPageTask = new KikaLetterPageTask(this, root, maxPages); + final Queue brandsAndChannel = new ConcurrentLinkedQueue<>(); + brandsAndChannel.addAll(aKikaLetterPageTask.fork().join()); + Log.sysLog("KIKA brandsAndChannel: " + brandsAndChannel.size()); + // lets take the channels and resolve them to all kinds + ConcurrentLinkedQueue channelQueue = brandsAndChannel.stream() + .filter(v -> v.getDocType().filter("channel"::equals).isPresent()) + .collect(Collectors.toCollection(ConcurrentLinkedQueue::new)); + final KikaChannelPageTask aKikaChannelPageTask = new KikaChannelPageTask(this, channelQueue, maxPages); + final Queue mixedKindsFromChannel = new ConcurrentLinkedQueue<>(); + mixedKindsFromChannel.addAll(aKikaChannelPageTask.fork().join()); + Log.sysLog("KIKA mixedKindsFromChannel: " + mixedKindsFromChannel.size()); + // lets use brands from letters and channels to resolve videoSubchannel + ConcurrentLinkedQueue brandQueue = Stream.concat(brandsAndChannel.stream(), mixedKindsFromChannel.stream()) + .filter(v -> v.getDocType().filter("broadcastSeries"::equals).isPresent()) + .collect(Collectors.toCollection(ConcurrentLinkedQueue::new)); + // add videoSubchannel and subchannel (same structure) from channels and add them to the result from brands + final ConcurrentLinkedQueue videoSubchannel = mixedKindsFromChannel.stream() + .filter(v -> v.getDocType().filter(t -> t.equals("videoSubchannel") || t.equals("subchannel")).isPresent()) + .map(v -> { + final String suffix = v.getDocType().filter("subchannel"::equals).isPresent() + ? "/teasers?page=0" + : "/videos?page=0&videoType=mainContent"; + return new KikaEntityDto( + v.getDocType(), v.getId(), v.getUuid(), v.getExternalId(), + v.getUrlPath(), v.getApiId(), + Optional.of(v.getUrl().replace("/relatedvideos/", "/videos/") + suffix), + v.getModificationDate()); + }) + .collect(Collectors.toCollection(ConcurrentLinkedQueue::new)); + Log.sysLog("KIKA brandQueue: " + brandQueue.size()); + // resolve brands from letters and channels to videoSubchannel + final KikaBrandPageTask aKikaBrandPageTask = new KikaBrandPageTask(this, brandQueue); + videoSubchannel.addAll(aKikaBrandPageTask.fork().join()); + Log.sysLog("KIKA videoSubchannel: " + videoSubchannel.size()); + // a videoSubchannel to video + final ConcurrentLinkedQueue videoAsset = new ConcurrentLinkedQueue<>(); + final KikaVideoSubchannelPageTask aKikaVideoSubchannelPageTask = new KikaVideoSubchannelPageTask(this, videoSubchannel, maxPages); + videoAsset.addAll(aKikaVideoSubchannelPageTask.fork().join()); + // add assets to the video + final ConcurrentLinkedQueue videoWithAssets = new ConcurrentLinkedQueue<>(); + final KikaAssetsPageTask aKikaAssetsPageTask = new KikaAssetsPageTask(this, videoAsset); + videoWithAssets.addAll(aKikaAssetsPageTask.fork().join()); + // + Log.sysLog("KIKA: Anzahl: " + videoWithAssets.size()); + meldungAddMax(videoWithAssets.size()); + + // convert video with assets to a film + return new KikaConverterTask(this, videoWithAssets); + } catch (final Exception ex) { + LOG.fatal("Exception in KIKA crawler.", ex); + } + + return null; + } + +} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaEntityDto.java b/src/main/java/mServer/crawler/sender/kika/KikaEntityDto.java new file mode 100644 index 000000000..081c7ed32 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/KikaEntityDto.java @@ -0,0 +1,73 @@ +package mServer.crawler.sender.kika; + +import mServer.crawler.sender.base.CrawlerUrlDTO; + +import java.util.Optional; + +public class KikaEntityDto extends CrawlerUrlDTO { + + Optional docType; // "broadcastSeries" + Optional id; // "die-abenteuer-von-awena-und-abduli-100" + Optional uuid; // "4e5071e7-4c38-4c69-84b4-3c9c3e8c49b1" + Optional externalId; // "content-63109" + Optional urlPath; // "/die-abenteuer-von-awena-und-abduli" + Optional apiId; // "die-abenteuer-von-awena-und-abduli-100" + //Optional url; // "https://www.kika.de/ackley/v1/brands/die-abenteuer-von-awena-und-abduli-100" + Optional modificationDate; //modificationDate "2026-05-04T21:02:41.605+02:00" + + public KikaEntityDto(Optional docType, Optional id, Optional uuid, + Optional externalId, Optional urlPath, Optional apiId, Optional url, + Optional modificationDate) { + super(url.get()); + this.docType = docType; + this.id = id; + this.uuid = uuid; + this.externalId = externalId; + this.urlPath = urlPath; + this.apiId = apiId; + this.modificationDate = modificationDate; + } + + public Optional getDocType() { + return docType; + } + + public Optional getId() { + return id; + } + + public Optional getUuid() { + return uuid; + } + + public Optional getExternalId() { + return externalId; + } + + public Optional getUrlPath() { + return urlPath; + } + + public Optional getApiId() { + return apiId; + } + + public Optional getModificationDate() { + return modificationDate; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof KikaEntityDto)) return false; + KikaEntityDto other = (KikaEntityDto) o; + return getUrl().equals(other.getUrl()); + } + + @Override + public int hashCode() { + return getUrl().hashCode(); + } + + +} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaFilmDto.java b/src/main/java/mServer/crawler/sender/kika/KikaFilmDto.java new file mode 100644 index 000000000..942e31c16 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/KikaFilmDto.java @@ -0,0 +1,72 @@ +package mServer.crawler.sender.kika; + +import java.util.List; +import java.util.Optional; + +public class KikaFilmDto extends KikaEntityDto{ + Optional date; + Optional title; + Optional teaserText; + Optional broadcastSeriesTitle; + Optional genDate; + Optional description; + Optional episodeNumber; + Optional durationInSeconds; + Optional season; + List assets; + public KikaFilmDto(Optional docType, Optional id, Optional uuid, Optional externalId, + Optional urlPath, Optional apiId, Optional url, Optional modificationDate, + Optional date, Optional title, Optional teaserText, Optional broadcastSeriesTitle, + Optional genDate, Optional description, Optional episodeNumber, + Optional durationInSeconds, Optional season) { + super(docType, id, uuid, externalId, urlPath, apiId, url, modificationDate); + this.date = date; + this.title = title; + this.teaserText = teaserText; + this.broadcastSeriesTitle = broadcastSeriesTitle; + this.genDate = genDate; + this.description = description; + this.episodeNumber = episodeNumber; + this.durationInSeconds = durationInSeconds; + this.season = season; + } + public List getAssets() { + return assets; + } + public void setAssets(List assets) { + this.assets = assets; + } + public Optional getDate() { + return date; + } + public Optional getTitle() { + return title; + } + public Optional getTeaserText() { + return teaserText; + } + public Optional getBroadcastSeriesTitle() { + return broadcastSeriesTitle; + } + public Optional getGenDate() { + return genDate; + } + public Optional getDescription() { + return description; + } + public Optional getEpisodeNumber() { + return episodeNumber; + } + public Optional getDurationInSeconds() { + return durationInSeconds; + } + public Optional getSeason() { + return season; + } + + + + + + +} diff --git a/src/main/java/mServer/crawler/sender/kika/Resolution.java b/src/main/java/mServer/crawler/sender/kika/Resolution.java deleted file mode 100644 index f1563c343..000000000 --- a/src/main/java/mServer/crawler/sender/kika/Resolution.java +++ /dev/null @@ -1,155 +0,0 @@ -package mServer.crawler.sender.kika; - -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.stream.Collectors; - -public enum Resolution { - HD(3, "HD"), NORMAL(2, "Normal"), SMALL(1, "Klein"), VERY_SMALL(0, "Sehr klein"); - - private enum CountingDirection { - HIGHER(+1), LOWER(-1); - - int direction; - - private CountingDirection(final int direction) { - this.direction = direction; - } - - public int getDirectionValue() { - return direction; - } - } - - /** - * The bigger the index the better the quality. - */ - private final int resolutionSize; - - private final String description; - - private Resolution(final int resolutionSize, final String description) { - this.resolutionSize = resolutionSize; - this.description = description; - } - - public static List getFromBestToLowest() { - return Arrays.asList(Resolution.values()).stream() - .sorted(Comparator.comparing(Resolution::getResolutionSize).reversed()) - .collect(Collectors.toList()); - } - - public static Resolution getHighestResolution() { - return Resolution.HD; - } - - public static Resolution getLowestResolution() { - return Resolution.VERY_SMALL; - } - - public static Resolution getNextHigherResolution(final Resolution startingResolution) { - return getNextResolutionByDirection(startingResolution, CountingDirection.HIGHER); - } - - public static Resolution getNextLowerResolution(final Resolution startingResolution) { - return getNextResolutionByDirection(startingResolution, CountingDirection.LOWER); - } - - /** - * Derzeit sind folgende ARD AudioVideo Ordinals bekannt:
- *
    - *
  • HD = 1280 width x 720 height
    • - *
  • Premium = 969 width x 540 height
    • - *
  • Large = 640 width x 360 height
    • - *
  • Standard = 512 width x 288 height
    • - *
  • Mobile = 480 width x 270 height
    • - *
  • Mobile_S = 320 width x 180 height
    • - *
- * - * @param profileName - * @return - */ - public static Resolution getResolutionFromArdAudioVideoOrdinalsByProfileName( - final String profileName) { - if (profileName.endsWith("HD")) { - return Resolution.HD; - } - if (profileName.endsWith("Premium")) { - return Resolution.NORMAL; - } - if (profileName.endsWith("Large")) { - return Resolution.SMALL; - } - if (profileName.endsWith("Standard")) { - return Resolution.VERY_SMALL; - } - if (profileName.endsWith("Mobile")) { - return Resolution.VERY_SMALL; - } - if (profileName.endsWith("Mobile_S")) { - return Resolution.VERY_SMALL; - } - - return Resolution.VERY_SMALL; - - } - - /** - * The following width size limits are relevant:
- *
    - *
  • HD = >= 1280 width
    • - *
  • Normal = >= 969 width
    • - *
  • Small = >= 640 width
    • - *
  • Very Small = < 640 width
    • - *
- * - * @param profileName - * @return - */ - public static Resolution getResolutionFromWidth(final int width) { - if (width >= 1280) { - return Resolution.HD; - } - if (width >= 969) { - return Resolution.NORMAL; - } - if (width >= 640) { - return Resolution.SMALL; - } - return Resolution.VERY_SMALL; - } - - static Resolution getNextResolutionByDirection(final Resolution startingResolution, - final CountingDirection direction) { - try { - return getResoultionByResolutionSize( - startingResolution.getResolutionSize() + direction.getDirectionValue()); - } catch (final NoSuchElementException nsee) { - return startingResolution; - } - } - - - static Resolution getResoultionByResolutionSize(final int searchedResolutionSize) { - for (final Resolution currentResolution : Resolution.values()) { - if (searchedResolutionSize == currentResolution.getResolutionSize()) { - return currentResolution; - } - } - - throw new NoSuchElementException( - String.format("Resolution with ResolutionIndex %d not found", searchedResolutionSize)); - } - - public String getDescription() { - return description; - } - - public int getResolutionSize() { - return resolutionSize; - } - - -} diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaApiTopicPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaApiTopicPageDeserializer.java deleted file mode 100644 index 4cce1ba31..000000000 --- a/src/main/java/mServer/crawler/sender/kika/json/KikaApiTopicPageDeserializer.java +++ /dev/null @@ -1,120 +0,0 @@ -package mServer.crawler.sender.kika.json; - -import com.google.gson.*; - -import mServer.crawler.sender.base.TopicUrlDTO; -import mServer.crawler.sender.base.JsonUtils; -import mServer.crawler.sender.base.UrlUtils; -import mServer.crawler.sender.kika.KikaApiConstants; -import mServer.crawler.sender.kika.KikaApiFilmDto; -import mServer.crawler.sender.kika.KikaApiTopicDto; - -import java.lang.reflect.Type; -import java.util.Optional; - - -public class KikaApiTopicPageDeserializer implements JsonDeserializer { - private static final String[] TAG_ERROR_CODE = new String[] {"error", "code"}; - private static final String[] TAG_ERROR_MESSAGE = new String[] {"error", "message"}; - private static final String[] TAG_NEXT_PAGE = new String[] {"_links", "next", "href"}; - private static final String[] TAG_FILM_ARRAY = new String[] {"_embedded","items"}; - private static final String[] TAG_BRAND_WEBSITE = new String[] {"_embedded", "brand", "homepageTeaserLinkUrl"}; - private static final String TAG_FILM_NAME = "title"; - private static final String TAG_FILM_ID = "id"; - private static final String TAG_FILM_EXPIRATIONDATE = "expirationDate"; - private static final String TAG_FILM_APPEARDATE = "appearDate"; - private static final String TAG_FILM_DESCRIPTION = "description"; - private static final String TAG_FILM_DATE = "date"; - private static final String TAG_FILM_DURATION = "duration"; - private static final String TAG_FILM_GEO = "geoProtection"; - private static final String[] TAG_FILM_TOPIC = new String[] {"_embedded","brand","title"}; - private static final String TAG_FILM_SOPHORAID = "sophoraId"; - private static final String TAG_FILM_TEASERIMAGEURL = "teaserImageUrl"; - // - - @Override - public KikaApiTopicDto deserialize( - final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context) - throws JsonParseException { - // - KikaApiTopicDto aKikaApiTopicDto = new KikaApiTopicDto(); - // catch error - Optional errorCode = JsonUtils.getElementValueAsString(jsonElement, TAG_ERROR_MESSAGE); - Optional errorMessage = JsonUtils.getElementValueAsString(jsonElement, TAG_ERROR_CODE); - if (errorCode.isPresent()) { - aKikaApiTopicDto.setError(errorCode, errorMessage); - return aKikaApiTopicDto; - } - // next page - Optional nextPage = JsonUtils.getElementValueAsString(jsonElement, TAG_NEXT_PAGE); - if (nextPage.isPresent()) { - aKikaApiTopicDto.setNextPage(new TopicUrlDTO("next page", UrlUtils.addProtocolIfMissing(KikaApiConstants.BASE_URL + nextPage.get(), UrlUtils.PROTOCOL_HTTPS))); - } - // film element - final JsonObject searchElement = jsonElement.getAsJsonObject(); - if (searchElement.has(TAG_FILM_ARRAY[0])) { - final JsonObject embeddedElement = searchElement.getAsJsonObject(TAG_FILM_ARRAY[0]); - if (embeddedElement.has(TAG_FILM_ARRAY[1])) { - final JsonArray itemArray = embeddedElement.getAsJsonArray(TAG_FILM_ARRAY[1]); - for (JsonElement arrayElement : itemArray) { - parseFilm(arrayElement).ifPresent(aKikaApiTopicDto::add); - } - } - } - return aKikaApiTopicDto; - } - - protected Optional parseFilm(final JsonElement arrayElement) { - Optional aFilm = Optional.empty(); - // - Optional oId = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_ID); - Optional oTitle = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_NAME); - Optional oDescription = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_DESCRIPTION); - Optional oDate = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_DATE); - Optional oDuration = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_DURATION); - Optional oGeo = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_GEO); - Optional oExpire = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_EXPIRATIONDATE); - Optional oAired = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_APPEARDATE); - Optional website = JsonUtils.getElementValueAsString(arrayElement, TAG_BRAND_WEBSITE); - Optional oTopic = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_TOPIC); - Optional oSophoraId = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_SOPHORAID); - Optional oTeaserImageUrl = JsonUtils.getElementValueAsString(arrayElement, TAG_FILM_TEASERIMAGEURL); - if (oSophoraId.isPresent() && oTeaserImageUrl.isPresent()) { - // try to reconstruct the website - website = reconstructWebsite(oTeaserImageUrl,oSophoraId); - } - // - if (oId.isPresent()) { - aFilm = Optional.of(new KikaApiFilmDto( - String.format(KikaApiConstants.FILM, oId.get()), - oTopic, - oTitle, - oId, - oDescription, - oDate, - oDuration, - oGeo, - oExpire, - oAired, - website - )); - } - return aFilm; - } - - protected Optional reconstructWebsite(Optional oTeaserImageUrl, Optional oSophoraId) { - String base = KikaApiConstants.WEBSITE; - // host wegschneiden - int start = oTeaserImageUrl.get().indexOf("/", "https://".length()+1); - // letzter Teil wegschneiden (das Bild vom teaserimage) - int stop = oTeaserImageUrl.get().lastIndexOf("/"); - // neue url mit host und mittelteil - base += oTeaserImageUrl.get().substring(start, stop); - // sophora hinzufügen - base += "/" + oSophoraId.get(); - // fix - base = base.replace("/sendungen/bilder/","/videos/"); - base = base.replace("/bilder/","/videos/"); - return Optional.of(base); - } -} diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaApiVideoInfoPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaApiVideoInfoPageDeserializer.java deleted file mode 100644 index 5bcf7dbab..000000000 --- a/src/main/java/mServer/crawler/sender/kika/json/KikaApiVideoInfoPageDeserializer.java +++ /dev/null @@ -1,112 +0,0 @@ -package mServer.crawler.sender.kika.json; - -import com.google.gson.*; - -import mServer.crawler.sender.base.JsonUtils; -import mServer.crawler.sender.base.UrlUtils; -import mServer.crawler.sender.kika.KikaApiVideoInfoDto; -import mServer.crawler.sender.kika.Resolution; - -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Optional; - - -public class KikaApiVideoInfoPageDeserializer implements JsonDeserializer { - // - private static final String[] TAG_ERROR_CODE = new String[] {"error", "code"}; - private static final String[] TAG_ERROR_MESSAGE = new String[] {"error", "message"}; - private static final String TAG_MP4_ASSETS_ARRAY = "hbbtvAssets"; - private static final String TAG_MP4_URL_TAG = "url"; - private static final String TAG_M3U8_ASSETS_ARRAY = "assets"; - private static final String TAG_M3U8_URL_TAG = "url"; - private static final String[] TAG_HAS_SUB = {"additional", "subtitle"}; - private static final String[] TAG_SUB_XML = {"additional", "closedCaption", "EBU-TT"}; - private static final String[] TAG_SUB_VTT = {"additional", "closedCaption", "WebVTT"}; - // - - @Override - public KikaApiVideoInfoDto deserialize( - final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context) - throws JsonParseException { - KikaApiVideoInfoDto aKikaApiVideoInfoDto = new KikaApiVideoInfoDto(); - // catch error - Optional errorCode = JsonUtils.getElementValueAsString(jsonElement, TAG_ERROR_MESSAGE); - Optional errorMessage = JsonUtils.getElementValueAsString(jsonElement, TAG_ERROR_CODE); - if (errorCode.isPresent()) { - aKikaApiVideoInfoDto.setError(errorCode, errorMessage); - return aKikaApiVideoInfoDto; - } - - // Video - addVideoUrlAndResolution(extractAllVideoUrls(jsonElement.getAsJsonObject()), aKikaApiVideoInfoDto); - // SUBTITLE - addSubtitle(jsonElement, aKikaApiVideoInfoDto); - return aKikaApiVideoInfoDto; - } - - private ArrayList extractAllVideoUrls(JsonObject root) { - // search for urls - final ArrayList urls = new ArrayList<>(); - // we will use the mp4 but in case missing - we will take the play list m3u8 - if (root.has(TAG_MP4_ASSETS_ARRAY)) { - final JsonArray mp4Array = root.get(TAG_MP4_ASSETS_ARRAY).getAsJsonArray(); - // - for (JsonElement arrayElement : mp4Array) { - final Optional url = JsonUtils.getElementValueAsString(arrayElement, TAG_MP4_URL_TAG); - if (url.isPresent() && url.get().endsWith("mp4")) { - urls.add(UrlUtils.addProtocolIfMissing(url.get(),UrlUtils.PROTOCOL_HTTPS)); - } - } - } - // no mp4 url found, lets check for m3u8 playlist entries - if (urls.isEmpty() && root.has(TAG_M3U8_ASSETS_ARRAY)) { - final JsonArray m3u8Array = root.get(TAG_M3U8_ASSETS_ARRAY).getAsJsonArray(); - // - for (JsonElement arrayElement : m3u8Array) { - final Optional url = JsonUtils.getElementValueAsString(arrayElement, TAG_M3U8_URL_TAG); - if (url.isPresent()) { - urls.add(UrlUtils.addProtocolIfMissing(url.get(),UrlUtils.PROTOCOL_HTTPS)); - } - } - } - return urls; - } - - private void addVideoUrlAndResolution(final ArrayList urls, KikaApiVideoInfoDto aKikaApiVideoInfoDto) { - // FIND BEST URL - // the last url in the list contain the highest quality, we will use it for HD - // the first url in the list contain the lowest quality, we will use it for SMALL - // gap into the middle (size/2) to take one of the medium quality urls - if (!urls.isEmpty()) { - if (urls.size() > 2) { - aKikaApiVideoInfoDto.addUrl(Resolution.HD, urls.getLast()); - } - if (urls.size() > 1) { - aKikaApiVideoInfoDto.addUrl(Resolution.NORMAL, urls.get(urls.size()/2)); - } - aKikaApiVideoInfoDto.addUrl(Resolution.SMALL, urls.getFirst()); - } - } - - private void addSubtitle(final JsonElement jsonElement, KikaApiVideoInfoDto aKikaApiVideoInfoDto) { - // SUBTITLE - Optional hasSubtitles = JsonUtils.getElementValueAsString(jsonElement, TAG_HAS_SUB); - if (hasSubtitles.isPresent() && hasSubtitles.get().equalsIgnoreCase("true")) { - // - aKikaApiVideoInfoDto.setSubtitle(true); - // - Optional subXml = JsonUtils.getElementValueAsString(jsonElement, TAG_SUB_XML); - if (subXml.isPresent()) { - aKikaApiVideoInfoDto.addSubtitle(subXml.get()); - } - // - Optional subvtt = JsonUtils.getElementValueAsString(jsonElement, TAG_SUB_VTT); - if (subvtt.isPresent()) { - aKikaApiVideoInfoDto.addSubtitle(subvtt.get()); - } - // - } - } - -} diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaAssetPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaAssetPageDeserializer.java new file mode 100644 index 000000000..36ed66062 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/json/KikaAssetPageDeserializer.java @@ -0,0 +1,69 @@ +package mServer.crawler.sender.kika.json; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import mServer.crawler.sender.base.JsonUtils; +import mServer.crawler.sender.kika.KikaAssetDto; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + + +public class KikaAssetPageDeserializer implements JsonDeserializer> { + private static final String TAG_ROOT = "assets"; + private static final String TAG_PROFILENAME = "profileName"; + private static final String TAG_FILENAME = "fileName"; + private static final String TAG_FILESIZE = "fileSize"; + private static final String TAG_MEDIATYPE = "mediaType"; + private static final String TAG_FRAMEWIDTH = "frameWidth"; + private static final String TAG_FRAMEHEIGHT = "frameHeight"; + private static final String TAG_BITRATEVIDEO = "bitrateVideo"; + private static final String TAG_BITRATEAUDIO = "bitrateAudio"; + private static final String TAG_TYPE = "type"; + private static final String TAG_URL = "url"; + private static final String TAG_VIDEOSUBTITLE = "videoSubtitle"; + private static final String TAG_WEBVTTURL = "webvttUrl"; + + @Override + public List deserialize( + final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context) + throws JsonParseException { + // + final List videos = new ArrayList<>(); + // film element + if (jsonElement.getAsJsonObject().has(TAG_ROOT) && + jsonElement.getAsJsonObject().get(TAG_ROOT).isJsonArray()) { + Optional videoSubtitle = JsonUtils.getElementValueAsString(jsonElement, TAG_VIDEOSUBTITLE); + Optional webvttUrl = JsonUtils.getElementValueAsString(jsonElement, TAG_WEBVTTURL); + for (JsonElement element : jsonElement.getAsJsonObject().getAsJsonArray(TAG_ROOT)) { + Optional e = parseElement(element, videoSubtitle, webvttUrl); + if (e.isPresent()) { + videos.add(e.get()); + } + } + } + return videos; + } + + protected Optional parseElement(JsonElement root, Optional videoSubtitle, Optional webvttUrl) { + KikaAssetDto e = new KikaAssetDto( + JsonUtils.getElementValueAsString(root, TAG_PROFILENAME), + JsonUtils.getElementValueAsString(root, TAG_FILENAME), + JsonUtils.getElementValueAsInteger(root, TAG_FILESIZE), + JsonUtils.getElementValueAsString(root, TAG_MEDIATYPE), + JsonUtils.getElementValueAsInteger(root, TAG_FRAMEWIDTH), + JsonUtils.getElementValueAsInteger(root, TAG_FRAMEHEIGHT), + JsonUtils.getElementValueAsInteger(root, TAG_BITRATEVIDEO), + JsonUtils.getElementValueAsInteger(root, TAG_BITRATEAUDIO), + JsonUtils.getElementValueAsString(root, TAG_TYPE), + JsonUtils.getElementValueAsString(root, TAG_URL), + videoSubtitle, + webvttUrl + ); + return Optional.of(e); + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaBrandPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaBrandPageDeserializer.java new file mode 100644 index 000000000..1c1d6445f --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/json/KikaBrandPageDeserializer.java @@ -0,0 +1,47 @@ +package mServer.crawler.sender.kika.json; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import mServer.crawler.sender.base.JsonUtils; +import mServer.crawler.sender.kika.KikaEntityDto; + +import java.lang.reflect.Type; +import java.util.Optional; + +public class KikaBrandPageDeserializer implements JsonDeserializer> { + private static final String TAG_TYPE = "docType"; + private static final String TAG_ID = "id"; + private static final String TAG_UUID = "uuid"; + private static final String TAG_EXTERNALID = "externalId"; + private static final String TAG_URL_PATH = "urlPath"; + private static final String TAG_MODIFICATIONDATE = "modificationDate"; + private static final String[] TAG_API_ID = new String[] {"videoSubchannel","id"}; + private static final String[] TAG_API_URL = new String[] {"videoSubchannel","videosPageUrl"}; + + + @Override + public Optional deserialize( + final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context) + throws JsonParseException { + // + if (JsonUtils.getElementValueAsString(jsonElement, TAG_API_URL).isPresent()) { + final KikaEntityDto e = new KikaEntityDto( + JsonUtils.getElementValueAsString(jsonElement, TAG_TYPE), + JsonUtils.getElementValueAsString(jsonElement, TAG_ID), + JsonUtils.getElementValueAsString(jsonElement, TAG_UUID), + JsonUtils.getElementValueAsString(jsonElement, TAG_EXTERNALID), + JsonUtils.getElementValueAsString(jsonElement, TAG_URL_PATH), + JsonUtils.getElementValueAsString(jsonElement, TAG_API_ID), + JsonUtils.getElementValueAsString(jsonElement, TAG_API_URL), + JsonUtils.getElementValueAsString(jsonElement, TAG_MODIFICATIONDATE) + ); + return Optional.of(e); + } + + return Optional.empty(); + } + + +} diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaChannelPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaChannelPageDeserializer.java new file mode 100644 index 000000000..480fd9466 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/json/KikaChannelPageDeserializer.java @@ -0,0 +1,71 @@ +package mServer.crawler.sender.kika.json; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import mServer.crawler.sender.base.JsonUtils; +import mServer.crawler.sender.base.PagedElementListDTO; +import mServer.crawler.sender.kika.KikaEntityDto; + +import java.lang.reflect.Type; +import java.util.Optional; + + +public class KikaChannelPageDeserializer implements JsonDeserializer> { + private static final String TAG_ROOT = "plusContent"; + private static final String TAG_BOXTYPE = "boxType"; + private static final String TAG_TYPE = "docType"; + private static final String TAG_ID = "id"; + private static final String TAG_UUID = "uuid"; + private static final String TAG_EXTERNALID = "externalId"; + private static final String TAG_URL_PATH = "urlPath"; + private static final String TAG_MODIFICATIONDATE = "modificationDate"; + private static final String[] TAG_API_ID = new String[] {"api","id"}; + private static final String[] TAG_API_URL = new String[] {"api","url"}; + private static final String[] TAG_NEXT_PAGE = new String[] {"links", "next"}; + + + @Override + public PagedElementListDTO deserialize( + final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context) + throws JsonParseException { + // + final PagedElementListDTO videoUrls = new PagedElementListDTO<>(); + // next page + Optional nextPage = JsonUtils.getElementValueAsString(jsonElement, TAG_NEXT_PAGE); + videoUrls.setNextPage(nextPage); + // + if (jsonElement.getAsJsonObject().has(TAG_ROOT) && + jsonElement.getAsJsonObject().get(TAG_ROOT).isJsonArray()) { + for (JsonElement plusContentElement : jsonElement.getAsJsonObject().getAsJsonArray(TAG_ROOT)) { + Optional boxType = JsonUtils.getElementValueAsString(plusContentElement, TAG_BOXTYPE); + if (boxType.orElse("").equalsIgnoreCase("MainVideo") || + boxType.orElse("").equalsIgnoreCase("RelatedVideo") || + boxType.orElse("").equalsIgnoreCase("Channel")) { + Optional channelLink = parseElement(plusContentElement.getAsJsonObject().get("channelLink")); + channelLink.ifPresent(videoUrls::addElement); + } + } + } + return videoUrls; + } + + protected Optional parseElement(JsonElement root) { + if (JsonUtils.getElementValueAsString(root, TAG_API_URL).isPresent()) { + KikaEntityDto e = new KikaEntityDto( + JsonUtils.getElementValueAsString(root, TAG_TYPE), + JsonUtils.getElementValueAsString(root, TAG_ID), + JsonUtils.getElementValueAsString(root, TAG_UUID), + JsonUtils.getElementValueAsString(root, TAG_EXTERNALID), + JsonUtils.getElementValueAsString(root, TAG_URL_PATH), + JsonUtils.getElementValueAsString(root, TAG_API_ID), + Optional.of(JsonUtils.getElementValueAsString(root, TAG_API_URL).get()), + JsonUtils.getElementValueAsString(root, TAG_MODIFICATIONDATE) + ); + return Optional.of(e); + } else { + return Optional.empty(); + } + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaLetterPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaLetterPageDeserializer.java new file mode 100644 index 000000000..7fc7f89c0 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/json/KikaLetterPageDeserializer.java @@ -0,0 +1,67 @@ +package mServer.crawler.sender.kika.json; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import mServer.crawler.sender.base.JsonUtils; +import mServer.crawler.sender.base.PagedElementListDTO; +import mServer.crawler.sender.kika.KikaEntityDto; + +import java.lang.reflect.Type; +import java.util.Optional; + + +public class KikaLetterPageDeserializer implements JsonDeserializer> { + private static final String TAG_ROOT = "content"; + private static final String TAG_TYPE = "docType"; + private static final String TAG_ID = "id"; + private static final String TAG_UUID = "uuid"; + private static final String TAG_EXTERNALID = "externalId"; + private static final String TAG_URL_PATH = "urlPath"; + private static final String TAG_MODIFICATIONDATE = "modificationDate"; + private static final String[] TAG_API_ID = new String[] {"api","id"}; + private static final String[] TAG_API_URL = new String[] {"api","url"}; + private static final String[] TAG_NEXT_PAGE = new String[] {"links", "next"}; + + + @Override + public PagedElementListDTO deserialize( + final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context) + throws JsonParseException { + // + final PagedElementListDTO videoUrls = new PagedElementListDTO<>(); + // next page + Optional nextPage = JsonUtils.getElementValueAsString(jsonElement, TAG_NEXT_PAGE); + videoUrls.setNextPage(nextPage); + // film element + if (jsonElement.getAsJsonObject().has(TAG_ROOT) && + jsonElement.getAsJsonObject().get(TAG_ROOT).isJsonArray()) { + for (JsonElement element : jsonElement.getAsJsonObject().getAsJsonArray(TAG_ROOT)) { + Optional e = parseElement(element); + if (e.isPresent()) { + videoUrls.addElement(e.get()); + } + } + } + return videoUrls; + } + + protected Optional parseElement(JsonElement root) { + if (JsonUtils.getElementValueAsString(root, TAG_API_URL).isPresent()) { + KikaEntityDto e = new KikaEntityDto( + JsonUtils.getElementValueAsString(root, TAG_TYPE), + JsonUtils.getElementValueAsString(root, TAG_ID), + JsonUtils.getElementValueAsString(root, TAG_UUID), + JsonUtils.getElementValueAsString(root, TAG_EXTERNALID), + JsonUtils.getElementValueAsString(root, TAG_URL_PATH), + JsonUtils.getElementValueAsString(root, TAG_API_ID), + JsonUtils.getElementValueAsString(root, TAG_API_URL), + JsonUtils.getElementValueAsString(root, TAG_MODIFICATIONDATE) + ); + return Optional.of(e); + } else { + return Optional.empty(); + } + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaVideoSubchannelPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaVideoSubchannelPageDeserializer.java new file mode 100644 index 000000000..72d9aa04c --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/json/KikaVideoSubchannelPageDeserializer.java @@ -0,0 +1,92 @@ +package mServer.crawler.sender.kika.json; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import mServer.crawler.sender.base.JsonUtils; +import mServer.crawler.sender.base.PagedElementListDTO; +import mServer.crawler.sender.kika.KikaFilmDto; + +import java.lang.reflect.Type; +import java.util.Optional; + + +public class KikaVideoSubchannelPageDeserializer implements JsonDeserializer> { + private static final String TAG_ROOT = "content"; + private static final String TAG_TYPE = "docType"; + private static final String TAG_ID = "id"; + private static final String TAG_UUID = "uuid"; + private static final String TAG_EXTERNALID = "externalId"; + private static final String TAG_URL_PATH = "urlPath"; + private static final String TAG_MODIFICATIONDATE = "modificationDate"; + private static final String[] TAG_API_ID = new String[] {"api","id"}; + private static final String[] TAG_API_URL = new String[] {"api","url"}; + // + private static final String TAG_DATE = "date"; + private static final String TAG_TITLE = "title"; + private static final String TAG_TEASER_TEXT = "teaserText"; + private static final String TAG_BROADCAST_SERIES_TITLE = "broadcastSeriesTitle"; + private static final String TAG_GEN_DATE = "genDate"; + private static final String[] TAG_DESCRIPTION = new String[] {"videoDetails","description"}; + private static final String[] TAG_EPISODE_NUMBER = new String[] {"videoDetails","episodeNumber"}; + private static final String[] TAG_DURATION_IN_SECONDS = new String[] {"videoDetails","durationInSeconds"}; + private static final String[] TAG_DURATION_IN_SECONDS2 = new String[] {"relatedVideoDetails","durationInSeconds"}; + private static final String[] TAG_SEASON = new String[] {"videoDetails","season"}; + + // + private static final String[] TAG_NEXT_PAGE = new String[] {"links", "next"}; + // + @Override + public PagedElementListDTO deserialize( + final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context) + throws JsonParseException { + // + final PagedElementListDTO videoUrls = new PagedElementListDTO<>(); + // next page + Optional nextPage = JsonUtils.getElementValueAsString(jsonElement, TAG_NEXT_PAGE); + videoUrls.setNextPage(nextPage); + // film element + if (jsonElement.getAsJsonObject().has(TAG_ROOT) && + jsonElement.getAsJsonObject().get(TAG_ROOT).isJsonArray()) { + for (JsonElement element : jsonElement.getAsJsonObject().getAsJsonArray(TAG_ROOT)) { + Optional e = parseElement(element); + if (e.isPresent()) { + videoUrls.addElement(e.get()); + } + } + } + return videoUrls; + } + + protected Optional parseElement(JsonElement root) { + if (JsonUtils.getElementValueAsString(root, TAG_API_URL).isPresent()) { + Optional durationInSeconds = JsonUtils.getElementValueAsString(root, TAG_DURATION_IN_SECONDS); + if (durationInSeconds.isEmpty()) { + durationInSeconds = JsonUtils.getElementValueAsString(root, TAG_DURATION_IN_SECONDS2); + } + KikaFilmDto e = new KikaFilmDto( + JsonUtils.getElementValueAsString(root, TAG_TYPE), + JsonUtils.getElementValueAsString(root, TAG_ID), + JsonUtils.getElementValueAsString(root, TAG_UUID), + JsonUtils.getElementValueAsString(root, TAG_EXTERNALID), + JsonUtils.getElementValueAsString(root, TAG_URL_PATH), + JsonUtils.getElementValueAsString(root, TAG_API_ID), + Optional.of(JsonUtils.getElementValueAsString(root, TAG_API_URL).get().replace("/relatedvideos/", "/videos/") + "/assets"), + JsonUtils.getElementValueAsString(root, TAG_MODIFICATIONDATE), + JsonUtils.getElementValueAsString(root, TAG_DATE), + JsonUtils.getElementValueAsString(root, TAG_TITLE), + JsonUtils.getElementValueAsString(root, TAG_TEASER_TEXT), + JsonUtils.getElementValueAsString(root, TAG_BROADCAST_SERIES_TITLE), + JsonUtils.getElementValueAsString(root, TAG_GEN_DATE), + JsonUtils.getElementValueAsString(root, TAG_DESCRIPTION), + JsonUtils.getElementValueAsString(root, TAG_EPISODE_NUMBER), + durationInSeconds, + JsonUtils.getElementValueAsString(root, TAG_SEASON) + ); + return Optional.of(e); + } else { + return Optional.empty(); + } + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java deleted file mode 100644 index 272fbb9b3..000000000 --- a/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java +++ /dev/null @@ -1,255 +0,0 @@ -package mServer.crawler.sender.kika.tasks; - -import com.google.common.util.concurrent.RateLimiter; -import com.google.gson.JsonDeserializer; -import com.google.gson.reflect.TypeToken; -import de.mediathekview.mlib.Const; -import de.mediathekview.mlib.daten.DatenFilm; -import de.mediathekview.mlib.tool.Log; -import jakarta.ws.rs.core.Response; -import mServer.crawler.CrawlerTool; -import mServer.crawler.sender.MediathekReader; -import mServer.crawler.sender.ard.ArdUrlOptimizer; -import mServer.crawler.sender.base.AbstractJsonRestTask; -import mServer.crawler.sender.base.AbstractRecursivConverterTask; -import mServer.crawler.sender.base.GeoLocations; -import mServer.crawler.sender.base.UrlUtils; -import mServer.crawler.sender.kika.KikaApiFilmDto; -import mServer.crawler.sender.kika.KikaApiVideoInfoDto; -import mServer.crawler.sender.kika.Resolution; -import mServer.crawler.sender.kika.json.KikaApiVideoInfoPageDeserializer; -import mServer.crawler.sender.zdf.ZdfVideoUrlOptimizer; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.Serial; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.concurrent.ConcurrentLinkedQueue; - -// extends AbstractRestTask -// return T Class from this task, desirialisation of class R , D , Reasearch in this url -public class KikaApiFilmTask extends AbstractJsonRestTask { - @Serial private static final long serialVersionUID = 1L; - private static final Logger LOG = LogManager.getLogger(KikaApiFilmTask.class); - private static final RateLimiter LIMITER = RateLimiter.create(15); - private transient ArdUrlOptimizer ardUrlOptimizer; - private transient ZdfVideoUrlOptimizer zdfVideoUrlOptimizer; - - public KikaApiFilmTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs) { - super(crawler, urlToCrawlDTOs, Optional.empty()); - ardUrlOptimizer = new ArdUrlOptimizer(); - zdfVideoUrlOptimizer = new ZdfVideoUrlOptimizer(); - } - - @Override - protected JsonDeserializer getParser(KikaApiFilmDto aDTO) { - return new KikaApiVideoInfoPageDeserializer(); - } - - @Override - protected Type getType() { - return new TypeToken() { - }.getType(); - } - - @Override - protected void handleHttpError(KikaApiFilmDto dto, URI url, Response response) { - LOG.fatal( - "A HTTP error {} occurred when getting REST information from: \"{}\".", - response.getStatus(), - url); - Log.errorLog(324978334, String.format("A HTTP error %d occurred when getting REST information from: \"%s\".", - response.getStatus(), - url)); - } - - @Override - protected void postProcessing(KikaApiVideoInfoDto aResponseObj, KikaApiFilmDto aDTO) { - // - LIMITER.acquire(); - // - if (aResponseObj.getErrorCode().isPresent()) { - LOG.error("Error {} : {} for target {} ", aResponseObj.getErrorCode().get(), aResponseObj.getErrorMesssage().orElse(""), aDTO.getUrl()); - Log.errorLog(324978335, String.format("Error %s} : %s for target %s ", aResponseObj.getErrorCode().get(), aResponseObj.getErrorMesssage().orElse(""), aDTO.getUrl())); - return; - } - // - if (aResponseObj.getVideoUrls().isEmpty()) { - LOG.error("No VideoUrls for {}", aDTO.getUrl()); - return; - } - // - final Optional airedDate = getAiredDateTime(aDTO); - if (aDTO.getTitle().isEmpty() || aDTO.getTopic().isEmpty() || airedDate.isEmpty() || aDTO.getDuration().isEmpty()) { - if (aDTO.getTitle().isEmpty()) { - LOG.error("Missing title for {}", aDTO.getUrl()); - } else if (aDTO.getTopic().isEmpty()) { - LOG.error("Missing topic for {}", aDTO.getUrl()); - } else if (airedDate.isEmpty()) { - LOG.error("Missing date for {}", aDTO.getUrl()); - } else if (aDTO.getDuration().isEmpty()) { - LOG.error("Missing duration for {}", aDTO.getUrl()); - } - return; - } - // - Map videoUrls = getVideoUrls(aResponseObj, aDTO); - Set subs = getSubtitle(aResponseObj, aDTO); - - DatenFilm aFilm = new DatenFilm( - Const.KIKA, - aDTO.getTopic().get(), - aDTO.getWebsite().orElse(""), - aDTO.getTitle().get(), - videoUrls.get(Resolution.NORMAL), - "", - airedDate.get().format(DateTimeFormatter.ofPattern("dd.MM.yyyy")), - airedDate.get().format(DateTimeFormatter.ofPattern("HH:mm:ss")), - parseDuration(aDTO, aDTO.getDuration()).get().getSeconds(), - aDTO.getDescription().orElse("") - ); - // - if (!subs.isEmpty()) { - CrawlerTool.addUrlSubtitle(aFilm, subs.toArray()[0].toString()); - } - // - if (videoUrls.containsKey(Resolution.SMALL)) { - CrawlerTool.addUrlKlein(aFilm, videoUrls.get(Resolution.SMALL)); - } - if (videoUrls.containsKey(Resolution.HD)) { - String url = videoUrls.get(Resolution.HD); - if (!videoUrls.get(Resolution.HD).contains("wdrmedien")) { - url = ardUrlOptimizer.optimizeHdUrl(url); - url = zdfVideoUrlOptimizer.getOptimizedUrlHd(url); - } - CrawlerTool.addUrlHd(aFilm, url); - } - // - getGeo(aDTO).ifPresent(geos -> { - geos.forEach(geo -> { - aFilm.arr[DatenFilm.FILM_GEO] = geo.getDescription(); - }); - }); - // - taskResults.add(aFilm); - } - - - protected Optional getAiredDateTime(KikaApiFilmDto aDTO) { - Optional airedDate; - if (aDTO.getDate().isPresent()) { - airedDate = parseLocalDateTime(aDTO, aDTO.getDate()); - } else { - airedDate = parseLocalDateTime(aDTO, aDTO.getAppearDate()); - } - return airedDate; - } - - protected Set getSubtitle(KikaApiVideoInfoDto aResponseObj, KikaApiFilmDto aDTO) { - Set urls = new HashSet<>(); - if (aResponseObj.hasSubtitle()) { - for (String subtitleUrlAsString : aResponseObj.getSubtitle()) { - try { - urls.add(new URL(UrlUtils.addProtocolIfMissing(subtitleUrlAsString, UrlUtils.PROTOCOL_HTTPS))); - } catch (MalformedURLException e) { - LOG.error("Invalid subtitle url {} for {} error {}", subtitleUrlAsString, aDTO.getUrl(), e); - } - } - if (aResponseObj.getSubtitle().isEmpty()) { - LOG.error("Missing subtitle for {}", aDTO.getUrl()); - } - } - return urls; - } - - protected Map getVideoUrls(KikaApiVideoInfoDto aResponseObj, KikaApiFilmDto aDTO) { - Map urls = new EnumMap<>(Resolution.class); - for (Map.Entry element : aResponseObj.getVideoUrls().entrySet()) { - urls.put(element.getKey(), element.getValue()); - } - return urls; - } - - protected Optional getWebsite(KikaApiFilmDto aDTO) { - Optional rs = Optional.empty(); - if (aDTO.getWebsite().isPresent()) { - try { - rs = Optional.of(new URL(aDTO.getWebsite().get())); - } catch (MalformedURLException e) { - LOG.error("Invalid website url {} for {} error {}", aDTO.getWebsite().get(), aDTO.getUrl(), e); - } - } - return rs; - } - - protected Optional> getGeo(KikaApiFilmDto aDTO) { - Optional> rs = Optional.empty(); - if (aDTO.getGeoProtection().isPresent()) { - Optional geo = parseGeo(aDTO, aDTO.getGeoProtection()); - if (geo.isPresent()) { - Collection collectionOfGeolocations = new ArrayList<>(); - collectionOfGeolocations.add(geo.get()); - rs = Optional.of(collectionOfGeolocations); - } - } - return rs; - } - - protected Optional parseLocalDateTime(KikaApiFilmDto sourceUrl, Optional text) { - Optional result = Optional.empty(); - if (text.isPresent()) { - try { - DateTimeFormatter formatter = DateTimeFormatter - .ofPattern("yyyy-MM-dd'T'HH:mm:ss"); - result = Optional.of(LocalDateTime.parse(text.get().substring(0, 19), formatter)); - } catch (Exception e) { - LOG.error("DateTimeFormatter failed for string {} url {} exception {}", text.get(), sourceUrl.getUrl(), e); - } - } - return result; - } - - // - protected Optional parseDuration(KikaApiFilmDto sourceUrl, Optional text) { - Optional result = Optional.empty(); - if (text.isPresent()) { - try { - int min = Integer.parseInt(text.get()); - result = Optional.of(Duration.ofSeconds(min)); - } catch (Exception e) { - LOG.error("Parse duration failed for string {} url {} exception {}", text.get(), sourceUrl.getUrl(), e); - } - } - return result; - } - - // - protected Optional parseGeo(KikaApiFilmDto sourceUrl, Optional text) { - Optional result = Optional.empty(); - if (text.isPresent()) { - if (text.get().equalsIgnoreCase("germany")) { - return Optional.of(GeoLocations.GEO_DE); - } else if (text.get().equalsIgnoreCase("worldwide")) { - return Optional.empty(); - } else { - LOG.error("Unknow GeoLocations {} url {}", text.get(), sourceUrl.getUrl()); - } - } - return result; - } - - @Override - protected AbstractRecursivConverterTask createNewOwnInstance( - ConcurrentLinkedQueue aElementsToProcess) { - return new KikaApiFilmTask(crawler, aElementsToProcess); - } - - -} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiTopicTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiTopicTask.java deleted file mode 100644 index 8e27228d0..000000000 --- a/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiTopicTask.java +++ /dev/null @@ -1,96 +0,0 @@ -package mServer.crawler.sender.kika.tasks; - -import com.google.gson.JsonDeserializer; -import com.google.gson.reflect.TypeToken; -import de.mediathekview.mlib.tool.Log; -import jakarta.ws.rs.core.Response; -import mServer.crawler.sender.MediathekReader; -import mServer.crawler.sender.base.AbstractJsonRestTask; -import mServer.crawler.sender.base.AbstractRecursivConverterTask; -import mServer.crawler.sender.kika.KikaApiFilmDto; -import mServer.crawler.sender.kika.KikaApiTopicDto; -import mServer.crawler.sender.kika.json.KikaApiTopicPageDeserializer; -import mServer.crawler.sender.base.TopicUrlDTO; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.Serial; -import java.lang.reflect.Type; -import java.net.URI; -import java.util.Optional; -import java.util.concurrent.ConcurrentLinkedQueue; - -// extends AbstractRestTask -// return T Class from this task, desirialisation of class R , D , Reasearch in this url -public class KikaApiTopicTask extends AbstractJsonRestTask { - @Serial private static final long serialVersionUID = 1L; - private static final Logger LOG = LogManager.getLogger(KikaApiTopicTask.class); - private int subPageIndex = 0; - private int maxPages = 1; - - public KikaApiTopicTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs, int subPageIndex, int maxPages) { - super(crawler, urlToCrawlDTOs, Optional.empty()); - this.subPageIndex = subPageIndex; - this.maxPages = maxPages; - } - - @Override - protected JsonDeserializer getParser(TopicUrlDTO aDTO) { - return new KikaApiTopicPageDeserializer(); - } - - @Override - protected Type getType() { - return new TypeToken() { - }.getType(); - } - - @Override - protected void postProcessing(KikaApiTopicDto aResponseObj, TopicUrlDTO aDTO) { - // - aResponseObj.getErrorCode().ifPresent(errorCode -> { - LOG.error("Error {} : {} for target {} ", errorCode, aResponseObj.getErrorMesssage().orElse(""), aDTO.getUrl()); - Log.errorLog(324978332, String.format("Error %s : %s for target %s ", errorCode, aResponseObj.getErrorMesssage().orElse(""), aDTO.getUrl())); - return; - }); - // - final Optional> subpageCrawler; - final Optional nextPageLink = aResponseObj.getNextPage(); - // - if (nextPageLink.isPresent() && maxPages > subPageIndex) { - final ConcurrentLinkedQueue nextPageLinks = new ConcurrentLinkedQueue<>(); - nextPageLinks.add(new TopicUrlDTO(aDTO.getTopic(), nextPageLink.get().getUrl())); // repack next page link to keep topic name - subpageCrawler = Optional.of(createNewOwnInstance(nextPageLinks)); - subpageCrawler.get().fork(); - } else { - subpageCrawler = Optional.empty(); - } - for (KikaApiFilmDto aFilm : aResponseObj.getElements()) { - if (aFilm.getTopic().isEmpty()) { - aFilm.setTopic(Optional.of(aDTO.getTopic())); - } - taskResults.add(aFilm); - } - // - subpageCrawler.ifPresent(nextPageCrawler -> taskResults.addAll(nextPageCrawler.join())); - } - - @Override - protected void handleHttpError(TopicUrlDTO dto, URI url, Response response) { - LOG.fatal( - "A HTTP error {} occurred when getting REST information from: \"{}\".", - response.getStatus(), - url); - Log.errorLog(324978333, String.format("A HTTP error %d occurred when getting REST information from: \"%s}\".", - response.getStatus(), - url)); - } - - @Override - protected AbstractRecursivConverterTask createNewOwnInstance( - ConcurrentLinkedQueue aElementsToProcess) { - return new KikaApiTopicTask(crawler, aElementsToProcess, subPageIndex + 1, maxPages); - } - - -} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaAssetsPageTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaAssetsPageTask.java new file mode 100644 index 000000000..21ed08721 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/tasks/KikaAssetsPageTask.java @@ -0,0 +1,65 @@ +package mServer.crawler.sender.kika.tasks; + +import com.google.gson.JsonDeserializer; +import com.google.gson.reflect.TypeToken; +import de.mediathekview.mlib.tool.Log; +import jakarta.ws.rs.core.Response; +import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.base.AbstractJsonRestTask; +import mServer.crawler.sender.base.AbstractRecursivConverterTask; +import mServer.crawler.sender.base.CrawlerUrlDTO; +import mServer.crawler.sender.kika.KikaAssetDto; +import mServer.crawler.sender.kika.KikaEntityDto; +import mServer.crawler.sender.kika.KikaFilmDto; +import mServer.crawler.sender.kika.json.KikaAssetPageDeserializer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.Type; +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class KikaAssetsPageTask + extends AbstractJsonRestTask, KikaFilmDto> { + private static final long serialVersionUID = 1L; + protected final transient Logger log = LogManager.getLogger(this.getClass()); + protected Optional> nextPageTask = Optional.empty(); + protected int maxSubpages; + + + public KikaAssetsPageTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs) { + super(crawler, urlToCrawlDTOs, Optional.empty()); + } + + @Override + protected JsonDeserializer> getParser(KikaFilmDto aDTO) { + return new KikaAssetPageDeserializer(); + } + + @Override + protected Type getType() { + return new TypeToken>() {}.getType(); + } + + @Override + protected void postProcessing(List aResponseObj, KikaFilmDto aDTO) { + if(!aResponseObj.isEmpty()) { + aDTO.setAssets(aResponseObj); + taskResults.add(aDTO); + } + } + + @Override + protected AbstractRecursivConverterTask createNewOwnInstance( + ConcurrentLinkedQueue aElementsToProcess) { + return new KikaAssetsPageTask(crawler, aElementsToProcess); + } + + @Override + protected void handleHttpError(KikaFilmDto dto, URI url, Response response) { + Log.errorLog(374323223, "http error " + response.getStatus() + ": " + url); + log.fatal("A HTTP error {} occurred when getting REST VideoInfo information from: \"{}\".", response.getStatus(), url); + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaBrandPageTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaBrandPageTask.java new file mode 100644 index 000000000..0df00e944 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/tasks/KikaBrandPageTask.java @@ -0,0 +1,62 @@ +package mServer.crawler.sender.kika.tasks; + +import com.google.gson.JsonDeserializer; +import com.google.gson.reflect.TypeToken; +import de.mediathekview.mlib.tool.Log; +import jakarta.ws.rs.core.Response; +import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.base.AbstractJsonRestTask; +import mServer.crawler.sender.base.AbstractRecursivConverterTask; +import mServer.crawler.sender.kika.KikaEntityDto; +import mServer.crawler.sender.kika.json.KikaBrandPageDeserializer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.Type; +import java.net.URI; +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class KikaBrandPageTask + extends AbstractJsonRestTask, KikaEntityDto> { + private static final long serialVersionUID = 1L; + protected final transient Logger log = LogManager.getLogger(this.getClass()); + protected Optional> nextPageTask = Optional.empty(); + protected int maxSubpages; + + + public KikaBrandPageTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs) { + super(crawler, urlToCrawlDTOs, Optional.empty()); + } + + @Override + protected JsonDeserializer> getParser(KikaEntityDto aDTO) { + return new KikaBrandPageDeserializer(); + } + + @Override + protected Type getType() { + return new TypeToken>() {}.getType(); + } + + @Override + protected void postProcessing(Optional aResponseObj, KikaEntityDto aDTO) { + if(aResponseObj.isPresent()) { + if(!taskResults.add(aResponseObj.get())) { + //log.debug("Duplicate KikaBrandPageTask {} from {}", aResponseObj.get().getUrl() , aDTO.getUrl()); + } + } + } + + @Override + protected AbstractRecursivConverterTask createNewOwnInstance( + ConcurrentLinkedQueue aElementsToProcess) { + return new KikaBrandPageTask(crawler, aElementsToProcess); + } + + @Override + protected void handleHttpError(KikaEntityDto dto, URI url, Response response) { + Log.errorLog(374323224, "http error " + response.getStatus() + ": " + url); + log.fatal("A HTTP error {} occurred when getting REST VideoInfo information from: \"{}\".", response.getStatus(), url); + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaChannelPageTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaChannelPageTask.java new file mode 100644 index 000000000..25bad622f --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/tasks/KikaChannelPageTask.java @@ -0,0 +1,90 @@ +package mServer.crawler.sender.kika.tasks; + +import com.google.gson.JsonDeserializer; +import com.google.gson.reflect.TypeToken; +import de.mediathekview.mlib.tool.Log; +import jakarta.ws.rs.core.Response; +import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.base.AbstractJsonRestTask; +import mServer.crawler.sender.base.AbstractRecursivConverterTask; +import mServer.crawler.sender.base.CrawlerUrlDTO; +import mServer.crawler.sender.base.PagedElementListDTO; +import mServer.crawler.sender.kika.KikaEntityDto; +import mServer.crawler.sender.kika.json.KikaChannelPageDeserializer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.Type; +import java.net.URI; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class KikaChannelPageTask + extends AbstractJsonRestTask, KikaEntityDto> { + private static final long serialVersionUID = 1L; + protected final transient Logger log = LogManager.getLogger(this.getClass()); + protected Optional> nextPageTask = Optional.empty(); + protected int maxSubpages; + + + public KikaChannelPageTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs, int maxSubpages) { + super(crawler, urlToCrawlDTOs, Optional.empty()); + this.maxSubpages = maxSubpages; + } + + @Override + protected JsonDeserializer> getParser(KikaEntityDto aDTO) { + return new KikaChannelPageDeserializer(); + } + + @Override + protected Type getType() { + return new TypeToken>() {}.getType(); + } + + protected void postProcessingNextPage(PagedElementListDTO aResponseObj, KikaEntityDto aDTO) { + if (aResponseObj.getNextPage().isEmpty()) { + return; + } + if (aResponseObj.getNextPage().get().contains("page="+maxSubpages+"&")) { + log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxSubpages); + return; + } + + final ConcurrentLinkedQueue nextPageLinks = new ConcurrentLinkedQueue<>(); + KikaEntityDto next = new KikaEntityDto(aDTO.getDocType(), aDTO.getId(), aDTO.getUuid(), aDTO.getExternalId(), aDTO.getUrlPath(), aDTO.getApiId(), aResponseObj.getNextPage(), aDTO.getModificationDate()); + nextPageLinks.add(next); + nextPageTask = Optional.of(createNewOwnInstance(nextPageLinks)); + nextPageTask.get().fork(); + //log.debug("started paging to url {} for {}", aResponseObj.getNextPage().get(), aDTO.getUrl()); + } + + protected void postProcessingElements(Set elements, CrawlerUrlDTO aDTO) { + for (KikaEntityDto element : elements) { + if(!taskResults.add(element)) { + //log.debug("Duplicate KikaChannelPageTask {} from {}", element.getUrl() , aDTO.getUrl()); + } + } + } + + @Override + protected void postProcessing(PagedElementListDTO aResponseObj, KikaEntityDto aDTO) { + postProcessingNextPage(aResponseObj, aDTO); + postProcessingElements(aResponseObj.getElements(), aDTO); + nextPageTask.ifPresent(paginationResults -> postProcessingElements(paginationResults.join(), aDTO)); + + } + + @Override + protected AbstractRecursivConverterTask createNewOwnInstance( + ConcurrentLinkedQueue aElementsToProcess) { + return new KikaChannelPageTask(crawler, aElementsToProcess, maxSubpages); + } + + @Override + protected void handleHttpError(KikaEntityDto dto, URI url, Response response) { + Log.errorLog(374323225, "http error " + response.getStatus() + ": " + url); + log.fatal("A HTTP error {} occurred when getting REST VideoInfo information from: \"{}\".", response.getStatus(), url); + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaConverterTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaConverterTask.java new file mode 100644 index 000000000..0800da428 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/tasks/KikaConverterTask.java @@ -0,0 +1,226 @@ +package mServer.crawler.sender.kika.tasks; + +import de.mediathekview.mlib.Const; +import de.mediathekview.mlib.daten.DatenFilm; +import de.mediathekview.mlib.tool.Log; +import mServer.crawler.CrawlerTool; +import mServer.crawler.FilmeSuchen; +import mServer.crawler.RunSender; +import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.ard.ArdUrlOptimizer; +import mServer.crawler.sender.base.AbstractRecursivConverterTask; +import mServer.crawler.sender.base.GeoLocations; +import mServer.crawler.sender.base.Qualities; +import mServer.crawler.sender.base.UrlUtils; +import mServer.crawler.sender.kika.KikaAssetDto; +import mServer.crawler.sender.kika.KikaConstants; +import mServer.crawler.sender.kika.KikaFilmDto; +import mServer.crawler.sender.zdf.ZdfVideoUrlOptimizer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; + +public class KikaConverterTask extends AbstractRecursivConverterTask { + private static final long serialVersionUID = 2437071037753218820L; + private static final Logger LOG = LogManager.getLogger(KikaConverterTask.class); + private final transient ZdfVideoUrlOptimizer zdfVideoUrlOptimizer; + private final transient ArdUrlOptimizer ardUrlOptimizer; + + public KikaConverterTask(MediathekReader aCrawler, ConcurrentLinkedQueue aUrlToCrawlDTOs) { + super(aCrawler, aUrlToCrawlDTOs); + zdfVideoUrlOptimizer = new ZdfVideoUrlOptimizer(); + ardUrlOptimizer = new ArdUrlOptimizer(); + } + + @Override + protected AbstractRecursivConverterTask createNewOwnInstance( + ConcurrentLinkedQueue aElementsToProcess) { + return new KikaConverterTask(crawler, aElementsToProcess); + } + + @Override + protected Integer getMaxElementsToProcess() { + return 50; + } + + @Override + protected void processElement(KikaFilmDto aElement) { + if (aElement.getAssets().isEmpty()) { + LOG.error("No VideoUrls for {}", aElement.getUrl()); + Log.errorLog(374323228, "no videourls for " + aElement.getUrl()); + FilmeSuchen.listeSenderLaufen.inc(crawler.getRunIdentifier(), RunSender.Count.FEHLER); + return; + } + final Optional airedDate = getAiredDateTime(aElement); + if (aElement.getTitle().isEmpty() || aElement.getBroadcastSeriesTitle().isEmpty() || aElement.getDate().isEmpty()) { + if (aElement.getTitle().isEmpty()) { + LOG.error("Missing title for {}", aElement.getUrl()); + } else if (aElement.getBroadcastSeriesTitle().isEmpty()) { + LOG.error("Missing topic for {}", aElement.getUrl()); + } else if (airedDate.isEmpty()) { + LOG.error("Missing date for {}", aElement.getUrl()); + } else if (aElement.getDurationInSeconds().isEmpty()) { + LOG.error("Missing duration for {}", aElement.getUrl()); + } + Log.errorLog(374323229, "missing title, topic, date or duration " + aElement.getUrl()); + FilmeSuchen.listeSenderLaufen.inc(crawler.getRunIdentifier(), RunSender.Count.FEHLER); + return; + } + + final Map videoUrls = getVideoUrls(aElement); + + String defaultUrl = Stream.of(Qualities.NORMAL, Qualities.HD, Qualities.SMALL) + .map(videoUrls::get) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + + DatenFilm aFilm = new DatenFilm( + Const.KIKA, + aElement.getBroadcastSeriesTitle().get(), + getWebsite(aElement).orElse(""), + aElement.getTitle().get().replace("DGS", "Gebärdensprache"), + defaultUrl, + "", + airedDate.get().format(DateTimeFormatter.ofPattern("dd.MM.yyyy")), + airedDate.get().format(DateTimeFormatter.ofPattern("HH:mm:ss")), + parseDuration(aElement, aElement.getDurationInSeconds()).get().getSeconds(), + aElement.getDescription().orElse("") + ); + // + final String sub = getSubtitle(aElement); + if (!sub.isEmpty()) { + CrawlerTool.addUrlSubtitle(aFilm, sub); + } + // + if (videoUrls.containsKey(Qualities.SMALL)) { + CrawlerTool.addUrlKlein(aFilm, videoUrls.get(Qualities.SMALL)); + } + if (videoUrls.containsKey(Qualities.HD)) { + String url = videoUrls.get(Qualities.HD); + if (!videoUrls.get(Qualities.HD).contains("wdrmedien")) { + url = ardUrlOptimizer.optimizeHdUrl(url); + url = zdfVideoUrlOptimizer.getOptimizedUrlHd(url); + } + CrawlerTool.addUrlHd(aFilm, url); + } + // + getGeo(aElement).ifPresent(geos -> { + geos.forEach(geo -> { + aFilm.arr[DatenFilm.FILM_GEO] = geo.getDescription(); + }); + }); + + if (!taskResults.add(aFilm)) { + //LOG.debug("Rejected duplicate {}",aFilm); + FilmeSuchen.listeSenderLaufen.inc(crawler.getRunIdentifier(), RunSender.Count.FEHLER); + } + } + // + protected Map getVideoUrls(KikaFilmDto aElement) { + Map urls = new EnumMap<>(Qualities.class); + for (KikaAssetDto element : aElement.getAssets()) { + if (element.getUrl().isPresent() && element.getResolution().isPresent()) { + String url = element.getUrl().get(); + Qualities res = element.getResolution().get(); + if (Qualities.HD.equals(res) && !url.contains("wdrmedien")) { + url = ardUrlOptimizer.optimizeHdUrl(url); + url = zdfVideoUrlOptimizer.getOptimizedUrlHd(url); + } + urls.put(res, url); + } + } + return urls; + } + // + protected Optional> getGeo(KikaFilmDto aDTO) { + Optional> rs = Optional.empty(); + if (!aDTO.getAssets().isEmpty()) { + Optional geo = parseGeo(aDTO); + if (geo.isPresent()) { + Collection collectionOfGeolocations = new ArrayList<>(); + collectionOfGeolocations.add(geo.get()); + rs = Optional.of(collectionOfGeolocations); + } + } + return rs; + } + protected Optional parseGeo(KikaFilmDto aDTO) { + Optional result = Optional.empty(); + String text = aDTO.getAssets().getFirst().getUrl().get(); + if (text.contains("/kika_de/") || text.contains("/de/") || + text.contains("/content-de/") || text.contains("kika_de-prod/geo/") || + text.contains("/progressive_geo/") + ) { + return Optional.of(GeoLocations.GEO_DE); + } else if (text.contains("/de-at-ch/") || text.contains("/dach/")) { + return Optional.of(GeoLocations.GEO_DE_AT_CH); + } else if (text.contains("/none/") || text.contains("/weltweit/")) { + return Optional.empty(); + } else { + //LOG.error("Unknow GeoLocations {} url {}", text, aDTO.getUrl()); + } + + return result; + } + // + protected Optional getAiredDateTime(KikaFilmDto aDTO) { + Optional airedDate; + if (aDTO.getDate().isPresent()) { + airedDate = parseLocalDateTime(aDTO, aDTO.getDate()); + } else { + airedDate = parseLocalDateTime(aDTO, aDTO.getGenDate()); + } + return airedDate; + } + + protected String getSubtitle(KikaFilmDto aElement) { + KikaAssetDto subForAllAssets = aElement.getAssets().getFirst(); + if (subForAllAssets.getVideoSubtitle().isPresent()) { + return UrlUtils.addProtocolIfMissing(subForAllAssets.getVideoSubtitle().get(), UrlUtils.PROTOCOL_HTTPS); + } + return ""; + } + // + protected Optional getWebsite(KikaFilmDto aDTO) { + Optional rs = Optional.empty(); + if (aDTO.getUrlPath().isPresent()) { + rs = Optional.of(KikaConstants.HOST + aDTO.getUrlPath().get() + "/" + aDTO.getApiId().get()); + } +return rs; + } + // + protected Optional parseLocalDateTime(KikaFilmDto sourceUrl, Optional text) { + Optional result = Optional.empty(); + if (text.isPresent()) { + try { + DateTimeFormatter formatter = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss"); + result = Optional.of(LocalDateTime.parse(text.get().substring(0, 19), formatter)); + } catch (Exception e) { + LOG.error("DateTimeFormatter failed for string {} url {} exception {}", text.get(), sourceUrl.getUrl(), e); + } + } + return result; + } + // + protected Optional parseDuration(KikaFilmDto sourceUrl, Optional text) { + Optional result = Optional.of(Duration.ZERO); + if (text.isPresent()) { + try { + int sec = Integer.parseInt(text.get()); + result = Optional.of(Duration.ofSeconds(sec)); + } catch (Exception e) { + LOG.error("Parse duration failed for string {} url {} exception {}", text.get(), sourceUrl.getUrl(), e); + } + } + return result; + } + +} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaLetterPageTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaLetterPageTask.java new file mode 100644 index 000000000..e1209ef04 --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/tasks/KikaLetterPageTask.java @@ -0,0 +1,89 @@ +package mServer.crawler.sender.kika.tasks; + +import com.google.gson.JsonDeserializer; +import com.google.gson.reflect.TypeToken; +import de.mediathekview.mlib.tool.Log; +import jakarta.ws.rs.core.Response; +import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.base.AbstractJsonRestTask; +import mServer.crawler.sender.base.AbstractRecursivConverterTask; +import mServer.crawler.sender.base.CrawlerUrlDTO; +import mServer.crawler.sender.base.PagedElementListDTO; +import mServer.crawler.sender.kika.KikaEntityDto; +import mServer.crawler.sender.kika.json.KikaLetterPageDeserializer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.Type; +import java.net.URI; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class KikaLetterPageTask + extends AbstractJsonRestTask, CrawlerUrlDTO> { + private static final long serialVersionUID = 1L; + protected final transient Logger log = LogManager.getLogger(this.getClass()); + protected Optional> nextPageTask = Optional.empty(); + protected int maxSubpages; + + + public KikaLetterPageTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs, int maxSubpages) { + super(crawler, urlToCrawlDTOs, Optional.empty()); + this.maxSubpages = maxSubpages; + } + + @Override + protected JsonDeserializer> getParser(CrawlerUrlDTO aDTO) { + return new KikaLetterPageDeserializer(); + } + + @Override + protected Type getType() { + return new TypeToken>() {}.getType(); + } + + protected void postProcessingNextPage(PagedElementListDTO aResponseObj, CrawlerUrlDTO aDTO) { + if (aResponseObj.getNextPage().isEmpty()) { + return; + } + if (aResponseObj.getNextPage().get().contains("page="+maxSubpages+"&")) { + log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxSubpages); + return; + } + + final ConcurrentLinkedQueue nextPageLinks = new ConcurrentLinkedQueue<>(); + nextPageLinks.add(new CrawlerUrlDTO(aResponseObj.getNextPage().get())); + nextPageTask = Optional.of(createNewOwnInstance(nextPageLinks)); + nextPageTask.get().fork(); + //log.debug("started paging to url {} for {}", aResponseObj.getNextPage().get(), aDTO.getUrl()); + } + + protected void postProcessingElements(Set elements, CrawlerUrlDTO aDTO) { + for (KikaEntityDto element : elements) { + if(!taskResults.add(element)) { + //log.debug("Duplicate KikaLetterPageTask {} from {}", element.getUrl() , aDTO.getUrl()); + } + } + } + + @Override + protected void postProcessing(PagedElementListDTO aResponseObj, CrawlerUrlDTO aDTO) { + postProcessingNextPage(aResponseObj, aDTO); + postProcessingElements(aResponseObj.getElements(), aDTO); + nextPageTask.ifPresent(paginationResults -> postProcessingElements(paginationResults.join(), aDTO)); + + } + + @Override + protected AbstractRecursivConverterTask createNewOwnInstance( + ConcurrentLinkedQueue aElementsToProcess) { + return new KikaLetterPageTask(crawler, aElementsToProcess, maxSubpages); + } + + @Override + protected void handleHttpError(CrawlerUrlDTO dto, URI url, Response response) { + Log.errorLog(374323226, "http error " + response.getStatus() + ": " + url); + log.fatal("A HTTP error {} occurred when getting REST VideoInfo information from: \"{}\".", response.getStatus(), url); + } +} diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaVideoSubchannelPageTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaVideoSubchannelPageTask.java new file mode 100644 index 000000000..3f5837cfd --- /dev/null +++ b/src/main/java/mServer/crawler/sender/kika/tasks/KikaVideoSubchannelPageTask.java @@ -0,0 +1,90 @@ +package mServer.crawler.sender.kika.tasks; + +import com.google.gson.JsonDeserializer; +import com.google.gson.reflect.TypeToken; +import de.mediathekview.mlib.tool.Log; +import jakarta.ws.rs.core.Response; +import mServer.crawler.sender.MediathekReader; +import mServer.crawler.sender.base.AbstractJsonRestTask; +import mServer.crawler.sender.base.AbstractRecursivConverterTask; +import mServer.crawler.sender.base.PagedElementListDTO; +import mServer.crawler.sender.kika.KikaEntityDto; +import mServer.crawler.sender.kika.KikaFilmDto; +import mServer.crawler.sender.kika.json.KikaVideoSubchannelPageDeserializer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.Type; +import java.net.URI; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class KikaVideoSubchannelPageTask + extends AbstractJsonRestTask, KikaEntityDto> { + private static final long serialVersionUID = 1L; + protected final transient Logger log = LogManager.getLogger(this.getClass()); + protected Optional> nextPageTask = Optional.empty(); + protected int maxSubpages; + + + public KikaVideoSubchannelPageTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs, int maxSubpages) { + super(crawler, urlToCrawlDTOs, Optional.empty()); + this.maxSubpages = maxSubpages; + } + + @Override + protected JsonDeserializer> getParser(KikaEntityDto aDTO) { + return new KikaVideoSubchannelPageDeserializer(); + } + + @Override + protected Type getType() { + return new TypeToken>() {}.getType(); + } + + protected void postProcessingNextPage(PagedElementListDTO aResponseObj, KikaEntityDto aDTO) { + if (aResponseObj.getNextPage().isEmpty()) { + return; + } + if (aResponseObj.getNextPage().get().contains("page="+maxSubpages+"&")) { + log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxSubpages); + return; + } + + final ConcurrentLinkedQueue nextPageLinks = new ConcurrentLinkedQueue<>(); + KikaEntityDto next = new KikaEntityDto(aDTO.getDocType(), aDTO.getId(), aDTO.getUuid(), aDTO.getExternalId(), aDTO.getUrlPath(), aDTO.getApiId(), aResponseObj.getNextPage(), aDTO.getModificationDate()); + nextPageLinks.add(next); + nextPageTask = Optional.of(createNewOwnInstance(nextPageLinks)); + nextPageTask.get().fork(); + //log.debug("started paging to url {} for {}", aResponseObj.getNextPage().get(), aDTO.getUrl()); + } + + protected void postProcessingElements(Set elements, KikaEntityDto aDTO) { + for (KikaFilmDto element : elements) { + if(!taskResults.add(element)) { + //log.debug("Duplicate KikaVideoSubchannelPageTask {} from {}", element.getUrl() , aDTO.getUrl()); + } + } + } + + @Override + protected void postProcessing(PagedElementListDTO aResponseObj, KikaEntityDto aDTO) { + postProcessingNextPage(aResponseObj, aDTO); + postProcessingElements(aResponseObj.getElements(), aDTO); + nextPageTask.ifPresent(paginationResults -> postProcessingElements(paginationResults.join(), aDTO)); + + } + + @Override + protected AbstractRecursivConverterTask createNewOwnInstance( + ConcurrentLinkedQueue aElementsToProcess) { + return new KikaVideoSubchannelPageTask(crawler, aElementsToProcess, maxSubpages); + } + + @Override + protected void handleHttpError(KikaEntityDto dto, URI url, Response response) { + Log.errorLog(374323227, "http error " + response.getStatus() + ": " + url); + log.fatal("A HTTP error {} occurred when getting REST VideoInfo information from: \"{}\".", response.getStatus(), url); + } +}