diff --git a/client-extras/src/main/java/nl/vpro/api/client/utils/PageUpdateApiUtil.java b/client-extras/src/main/java/nl/vpro/api/client/utils/PageUpdateApiUtil.java index f7a7c18a..11be10c1 100644 --- a/client-extras/src/main/java/nl/vpro/api/client/utils/PageUpdateApiUtil.java +++ b/client-extras/src/main/java/nl/vpro/api/client/utils/PageUpdateApiUtil.java @@ -27,8 +27,7 @@ import nl.vpro.domain.media.MediaObject; import nl.vpro.domain.page.Page; import nl.vpro.domain.page.PageIdMatch; -import nl.vpro.domain.page.update.DeleteResult; -import nl.vpro.domain.page.update.PageUpdate; +import nl.vpro.domain.page.update.*; import nl.vpro.jackson2.Jackson2Mapper; import nl.vpro.rs.client.Utils; @@ -79,23 +78,12 @@ protected Result save(@NotNull @Valid PageUpdate update, boolean wait) { limiter.acquire(); try { - Result result = handleResponse( - pageUpdateApiClient.getPageUpdateRestService().save(update, wait), - update, STRING, String.class - ); - // temporary, later we may do Result - if (result.isOk()) { - log.info(result.getEntity()); - return Result.builder().status(result.getStatus()).build(); - } else { - if ((! retryErrors) || (! result.needsRetry())) { - log.warn(result.getErrors()); - return Result.builder().status(result.getStatus()).errors(result.getErrors()).build(); - } - } - limiter.setCurrentRateToMinRate(); - log.warn("Retrying {}", update); + SaveResult result = pageUpdateApiClient.getPageUpdateRestService().save(update, wait); + limiter.upRate(); + return Result.success(null); + } catch (ProcessingException e) { + limiter.downRate(); return exceptionToResult(e); } } @@ -108,50 +96,33 @@ public PageUpdate get(@NotNull String url) { return Utils.wrapNotFound(() -> pageUpdateApiClient.getPageUpdateRestService().load(url, false, match)).orElse(null); } - public Result delete(@NotNull String id) { + public DeleteResult delete(@NotNull String id) { limiter.acquire(); PageIdMatch match = id.startsWith("crid:") ? PageIdMatch.CRID : PageIdMatch.URL; - try { - return handleResponse( - pageUpdateApiClient.getPageUpdateRestService() - .delete(id, false, 1, false, match, null, null), id, STRING, DeleteResult.class - ); - } catch (ProcessingException e) { - return exceptionToResult(e); - } + return pageUpdateApiClient.getPageUpdateRestService() + .delete(id, false, 1, false, match); + } - public Result deleteWhereStartsWith(@NotNull String prefix) { + public DeleteResult deleteWhereStartsWith(@NotNull String prefix) { limiter.acquire(); PageIdMatch match = prefix.startsWith("crid:") ? PageIdMatch.CRID : PageIdMatch.URL; int batchSize = 10000; - try { - DeleteResult result = null; - while (true) { - Result r = handleResponse( - pageUpdateApiClient.getPageUpdateRestService() - .delete(prefix, true, batchSize, true, match, null,null), prefix, STRING, DeleteResult.class - ); - log.info("Batch deleted {}: {}", prefix, r); - if (result == null) { - result = r.getEntity(); - } else { - result = result.and(r.getEntity()); - } - if (r.isOk()) { - if (r.getEntity().getCount() == 0) { - return Result.builder() - .entity(result) - .status(Result.Status.SUCCESS) - .build(); - } - } + DeleteResult result = null; + while (true) { + DeleteResult r = pageUpdateApiClient.getPageUpdateRestService() + .delete(prefix, true, batchSize, true, match); + log.info("Batch deleted {}: {}", prefix, r); + if (result != null) { + result = result.and(r); + } else { + result = r; + } + if (r.getCount() == 0) { + return result; } - - } catch (ProcessingException e) { - return exceptionToResult(e); } } diff --git a/client-extras/src/test/java/nl/vpro/api/client/utils/PageUpdateApiClientUtilTest.java b/client-extras/src/test/java/nl/vpro/api/client/utils/PageUpdateApiClientUtilTest.java index 1367e4d2..0ce5b2f2 100644 --- a/client-extras/src/test/java/nl/vpro/api/client/utils/PageUpdateApiClientUtilTest.java +++ b/client-extras/src/test/java/nl/vpro/api/client/utils/PageUpdateApiClientUtilTest.java @@ -43,32 +43,33 @@ public void testSaveInvalid() { @Test public void testDelete() { String id = "http://BESTAATNIET"; - Result result = util.delete(id); - assertThat(result.getStatus()).isEqualTo(Result.Status.SUCCESS); - assertThat(result.getErrors()).isNull(); + DeleteResult result = util.delete(id); + log.info("{}", result); } @Test public void accesDenied() { - String willCauseDeny = "\n" + - "\n" + - " crid://vpro/media/cultura/RBX_NTR_632457\n" + - " NTR\n" + - " Blaudzun te gast met zijn favoriete platen\n" + - " \n" + - " \n" + - " Blaudzun te gast met zijn favoriete platen\n" + - " Een grote bril, een ietwat apart kapsel en een gitaar. Dat is Johannes Sigmond uit Arnhem, ook wel bekend als Blaudzun. Blaudzun was een gegarandeerd feest op elk festival van afgelopen zomer! Vanavond komt deze singer-songwriter naar Winfrieds Woonkamer om zijn favoriete platen aan jou te laten horen. Zijn lijst van dit uur:1. The Jig - Bike Ride2. Prince - I Would Die 4 You3. Child Of Lov ft Damon Albarn - One Day4. Anne Murray - Paths Of Victory5. Outkast - Roses6. Mulatu Astatke - Yekermo Sew7. Earl Sweatshirt - Burgundy 8. Nina Simone - Sinnerman9. TLC - Waterfalls10. The Roots - Singing Man\n" + - " \n" + - " \n" + - " \n" + - " Blaudzun_-_2012-05-05.jpg\n" + - " Blaudzun te gast met zijn favoriete platen\n" + - " \n" + - " http://images.poms.omroep.nl/image/s360/289017.jpg\n" + - " \n" + - " \n" + - "\n"; + String willCauseDeny = """ + + + crid://vpro/media/cultura/RBX_NTR_632457 + NTR + Blaudzun te gast met zijn favoriete platen + + + Blaudzun te gast met zijn favoriete platen + Een grote bril, een ietwat apart kapsel en een gitaar. Dat is Johannes Sigmond uit Arnhem, ook wel bekend als Blaudzun. Blaudzun was een gegarandeerd feest op elk festival van afgelopen zomer! Vanavond komt deze singer-songwriter naar Winfrieds Woonkamer om zijn favoriete platen aan jou te laten horen. Zijn lijst van dit uur:1. The Jig - Bike Ride2. Prince - I Would Die 4 You3. Child Of Lov ft Damon Albarn - One Day4. Anne Murray - Paths Of Victory5. Outkast - Roses6. Mulatu Astatke - Yekermo Sew7. Earl Sweatshirt - Burgundy 8. Nina Simone - Sinnerman9. TLC - Waterfalls10. The Roots - Singing Man + + + + Blaudzun_-_2012-05-05.jpg + Blaudzun te gast met zijn favoriete platen + + http://images.poms.omroep.nl/image/s360/289017.jpg + + + + """; //System.out.println(willCauseError); PageUpdate update = JAXB.unmarshal(new StringReader(willCauseDeny), PageUpdate.class); Result result = util.save(update); @@ -79,7 +80,7 @@ public void accesDenied() { @Test public void deleteWhereStartsWith() { log.info("{}", util); - Result result = util.deleteWhereStartsWith("http://bla/bloe"); - log.info("{}", result.getEntity()); + DeleteResult result = util.deleteWhereStartsWith("http://bla/bloe"); + log.info("{}", result); } } diff --git a/media-backend-api-client/src/main/java/nl/vpro/api/client/media/MediaRestClient.java b/media-backend-api-client/src/main/java/nl/vpro/api/client/media/MediaRestClient.java index 78b8d334..dcaca74f 100644 --- a/media-backend-api-client/src/main/java/nl/vpro/api/client/media/MediaRestClient.java +++ b/media-backend-api-client/src/main/java/nl/vpro/api/client/media/MediaRestClient.java @@ -95,6 +95,7 @@ public class MediaRestClient extends AbstractApiClient implements MediaRestClien public static final TriFunction DEFAULT_HEADER_LEVEL = (m, a, s) -> s.equals(Headers.NPO_WARNING_HEADER) ? Level.WARN : Level.DEBUG; + @Setter private int defaultMax = 50; private final RateLimiter throttle = RateLimiter.create(1.0); @@ -108,6 +109,7 @@ public class MediaRestClient extends AbstractApiClient implements MediaRestClien private FrameCreatorRestService frameCreatorRestService; + @Setter private Map headers; Supplier version; @@ -130,7 +132,6 @@ public class MediaRestClient extends AbstractApiClient implements MediaRestClien @Setter private boolean validateInput = false; - @Getter @Setter private AssemblageConfig.Steal stealCrids = AssemblageConfig.Steal.IF_DELETED; @@ -389,10 +390,6 @@ public void setUserNamePassword(String semicolonSeperated) { } } - public void setHeaders(Map headers) { - this.headers = headers; - } - public String getVersion() { if (version == null) { version = () -> Suppliers.memoizeWithExpiration(() -> { @@ -725,18 +722,14 @@ public String set(MediaUpdate mediaUpdate, String errors) { public Iterator getAllMembers(String mid) { return BatchedReceiver.builder() .batchSize(240) - .batchGetter((offset, max) -> { - return getBackendRestService().getGroupMembers(EntityType.NoSegments.media, mid, offset, max, ASC, followMerges, owner, deletes).iterator(); - }) + .batchGetter((offset, max) -> getBackendRestService().getGroupMembers(EntityType.NoSegments.media, mid, offset, max, ASC, followMerges, owner, deletes).iterator()) .build(); } public Iterator getAllEpisodes(String mid) { return BatchedReceiver.builder() .batchSize(defaultMax) - .batchGetter((offset, max) -> { - return getBackendRestService().getGroupEpisodes(mid, offset, max, ASC, followMerges, owner, deletes).iterator(); - }) + .batchGetter((offset, max) -> getBackendRestService().getGroupEpisodes(mid, offset, max, ASC, followMerges, owner, deletes).iterator()) .build(); } @@ -744,18 +737,14 @@ public Iterator getAllEpisodes(String mid) { public Iterator getAllFullMembers(String mid) { return BatchedReceiver.builder() .batchSize(defaultMax) - .batchGetter((offset, max) -> { - return getBackendRestService().getFullGroupMembers(EntityType.NoSegments.media, mid, offset, max, ASC, followMerges, deletes).iterator(); - }) + .batchGetter((offset, max) -> getBackendRestService().getFullGroupMembers(EntityType.NoSegments.media, mid, offset, max, ASC, followMerges, deletes).iterator()) .build(); } public Iterator getAllFullEpisodes(String mid) { return BatchedReceiver.builder() .batchSize(defaultMax) - .batchGetter((offset, max) -> { - return getBackendRestService().getFullGroupEpisodes(mid, offset, max, ASC, followMerges, deletes).iterator(); - }) + .batchGetter((offset, max) -> getBackendRestService().getFullGroupEpisodes(mid, offset, max, ASC, followMerges, deletes).iterator()) .build(); } @@ -784,10 +773,6 @@ public String deleteSubtitles(SubtitlesId id) { } } - public void setDefaultMax(int max) { - this.defaultMax = max; - } - public double getThrottleRate() { return this.throttle.getRate(); diff --git a/pages-backend-api-client/pom.xml b/pages-backend-api-client/pom.xml index 17d4d86b..adba7681 100644 --- a/pages-backend-api-client/pom.xml +++ b/pages-backend-api-client/pom.xml @@ -24,6 +24,10 @@ nl.vpro.shared vpro-shared-client-resteasy + + nl.vpro.shared + vpro-shared-rs-client + nl.vpro.pages pages-rs-services @@ -41,11 +45,5 @@ org.projectlombok lombok - - - jakarta.servlet - jakarta.servlet-api - provided - diff --git a/pages-backend-api-client/src/main/java/nl/vpro/api/client/pages/PageUpdateApiClient.java b/pages-backend-api-client/src/main/java/nl/vpro/api/client/pages/PageUpdateApiClient.java index 45d01b2f..fc02959f 100644 --- a/pages-backend-api-client/src/main/java/nl/vpro/api/client/pages/PageUpdateApiClient.java +++ b/pages-backend-api-client/src/main/java/nl/vpro/api/client/pages/PageUpdateApiClient.java @@ -332,7 +332,8 @@ public String toString() { @Override protected void buildResteasy(ResteasyClientBuilder builder) { builder - .register(authentication); + .register(authentication) + ; } diff --git a/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/PageUpdateApiClientITest.java b/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/PageUpdateApiClientITest.java index ae84d787..fb6d717f 100644 --- a/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/PageUpdateApiClientITest.java +++ b/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/PageUpdateApiClientITest.java @@ -1,11 +1,11 @@ package nl.vpro.api.client.pages; +import lombok.extern.log4j.Log4j2; + import java.time.Instant; -import javax.ws.rs.core.Response; import javax.xml.bind.JAXB; -import org.jboss.resteasy.api.validation.ViolationReport; import org.junit.jupiter.api.*; import nl.vpro.domain.classification.ClassificationService; @@ -19,6 +19,7 @@ @Disabled("This required running server at publish-test") +@Log4j2 public class PageUpdateApiClientITest { private static PageUpdateApiClient clients; @@ -39,15 +40,8 @@ public void testSave() { PageUpdate instance = PageUpdateBuilder.article("http://www.meeuw.org/test/1234") .title("my title " + Instant.now()) .broadcasters("VPRO").build(); - try (Response response = client.save(instance, null)) { - if (response.getStatus() == 400) { - ViolationReport report = response.readEntity(ViolationReport.class); - JAXB.marshal(report, System.out); - JAXB.marshal(instance, System.out); - - } - assertEquals(202, response.getStatus()); - } + SaveResult saveResult = client.save(instance, null); + log.info("{}", saveResult); } @@ -61,15 +55,8 @@ public void testSaveTopStory() { .build(); JAXB.marshal(page, System.out); - try (Response response = client.save(page, null)) { - if (response.getStatus() == 400) { - ViolationReport report = response.readEntity(ViolationReport.class); - JAXB.marshal(report, System.out); - JAXB.marshal(page, System.out); - - } - assertEquals(202, response.getStatus()); - } + SaveResult result = client.save(page, null); + log.info("{}", result); } @@ -82,32 +69,16 @@ public void testSaveWithTopStory() { .title("Page with topstory (" + Instant.now() + ")") .links(LinkUpdate.topStory("http://www.meeuw.org/test/topstory", "heel goed artikel")) .build(); - try (Response response = client.save(page, null)) { - JAXB.marshal(page, System.out); - - if (response.getStatus() == 400) { - ViolationReport report = response.readEntity(ViolationReport.class); - JAXB.marshal(report, System.out); - - } - assertEquals(202, response.getStatus()); - } - + SaveResult result = client.save(page, null); + log.info("{}", result); } @Test public void testDelete() { PageUpdateRestService client = clients.getPageUpdateRestService(); - try (Response response = client.delete("http://www.meeuw.org/test/1234", false, 1, true, PageIdMatch.URL, null, null)) { - - if (response.getStatus() == 400) { - ViolationReport report = response.readEntity(ViolationReport.class); - JAXB.marshal(report, System.out); - - } - assertEquals(202, response.getStatus()); - } + DeleteResult deleteResult = client.delete("http://www.meeuw.org/test/1234", false, 1, true, PageIdMatch.URL); + log.info("{}", deleteResult); } @@ -115,15 +86,8 @@ public void testDelete() { @Test public void testDeleteMultiple() { PageUpdateRestService client = clients.getPageUpdateRestService(); - try (Response response = client.delete("http://www.meeuw.org/", true, 100, true, PageIdMatch.URL, null, null)) { - if (response.getStatus() == 400) { - ViolationReport report = response.readEntity(ViolationReport.class); - JAXB.marshal(report, System.out); - - } - assertEquals(202, response.getStatus()); - } - + DeleteResult result = client.delete("http://www.meeuw.org/", true, 100, true, PageIdMatch.URL); + log.info("{}", result); } @Test diff --git a/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/ReindexFromCouchdb.java b/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/ReindexFromCouchdb.java index 759f40e5..6a4695d4 100644 --- a/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/ReindexFromCouchdb.java +++ b/pages-backend-api-client/src/test/java/nl/vpro/api/client/pages/ReindexFromCouchdb.java @@ -2,8 +2,6 @@ import java.io.*; -import javax.ws.rs.core.Response; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -74,12 +72,8 @@ public void reindex() throws IOException { } count++; System.out.println(url); - Response response = restService.save(pu, false); - response.close(); - if (response.getStatus() != 202) { - System.out.println("" + response); - } - + SaveResult saveResult = restService.save(pu, false); + System.out.println(saveResult); } } } diff --git a/pom.xml b/pom.xml index 3722cf82..1cf98e1b 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ 17 - 7.10.1-SNAPSHOT + 7.11-SNAPSHOT