Merge branch 'dev'

app/build.gradle

@@ -38,7 +38,7 @@ android {
         minSdkVersion 23
         targetSdkVersion 34
         versionCode 130 // is updated automatically by BitRise; only used when building locally
-        versionName '1.18.6'
+        versionName '1.18.7'
 
         def includeObjectBoxBrowser = System.getenv("INCLUDE_OBJECTBOX_BROWSER") ?: "false"
         def includeLeakCanary = System.getenv("INCLUDE_LEAK_CANARY") ?: "false"

app/src/main/java/me/devsaki/hentoid/activities/sources/HitomiActivity.java

@@ -30,7 +30,7 @@
 public class HitomiActivity extends BaseWebActivity {
 
     private static final String DOMAIN_FILTER = "hitomi.la";
-    private static final String[] GALLERY_FILTER = {"//hitomi.la/[manga|doujinshi|gamecg|cg]+/[^/]+-[0-9]{2,}.html(#[0-9]{1,2}){0,1}$"};
+    private static final String[] GALLERY_FILTER = {"//hitomi.la/[manga|doujinshi|gamecg|cg|imageset]+/[^/]+-[0-9]{2,}.html(#[0-9]{1,2}){0,1}$"};
     private static final String[] RESULTS_FILTER = {"//hitomi.la[/]{0,1}$", "//hitomi.la[/]{0,1}\\?", "//hitomi.la/search.html", "//hitomi.la/index-[\\w%\\-\\.\\?]+", "//hitomi.la/(series|artist|tag|character)/[\\w%\\-\\.\\?]+"};
     private static final String[] BLOCKED_CONTENT = {"hitomi-horizontal.js", "hitomi-vertical.js", "invoke.js", "ion.sound"};
     private static final String[] JS_URL_PATTERN_WHITELIST = {"//hitomi.la[/]{0,1}$", "galleries/[\\w%\\-]+.js$", "//hitomi.la/[?]page=[0-9]+"};

app/src/main/java/me/devsaki/hentoid/database/ObjectBoxDAO.java

@@ -547,21 +547,21 @@ private Group enrichGroupWithItemsByQuery(@NonNull final Group g) {
 
     private Group enrichCustomGroups(@NonNull final Group g) {
         if (g.grouping.equals(Grouping.CUSTOM)) {
+            List<GroupItem> newItems;
             if (g.isUngroupedGroup()) { // Populate Ungrouped custom group
-                List<GroupItem> items = Stream.of(db.selectUngroupedContentIds()).map(id -> new GroupItem(id, g, -1)).toList();
-                g.setItems(items);
-                if (!items.isEmpty()) {
-                    Content c = selectContent(items.get(0).getContentId());
-                    g.coverContent.setTarget(c);
-                }
+                newItems = Stream.of(db.selectUngroupedContentIds()).map(id -> new GroupItem(id, g, -1)).toList();
             } else { // Reselect items; only take items from the library to avoid counting those who've been sent back to the Queue
                 long[] groupContent = db.selectContentIdsByGroup(g.id); // Specific query to get there fast
-                List<GroupItem> newItems = new ArrayList<>();
+                newItems = new ArrayList<>();
                 for (int i = 0; i < groupContent.length; i++) {
                     newItems.add(new GroupItem(groupContent[i], g, i));
                 }
-                g.setItems(newItems);
-                if (!newItems.isEmpty()) {
+            }
+            g.setItems(newItems);
+            // Reset cover content if it isn't among remaining books
+            if (!newItems.isEmpty()) {
+                List<Long> newContents = Stream.of(newItems).map(GroupItem::getContentId).toList();
+                if (!newContents.contains(g.coverContent.getTargetId())) {
                     Content c = selectContent(newItems.get(0).getContentId());
                     g.coverContent.setTarget(c);
                 }

app/src/main/java/me/devsaki/hentoid/enums/Site.java

@@ -56,6 +56,7 @@ public enum Site {
             NEXUS, // Dead
             HBROWSE, // Dead
             HENTAICAFE, // Removed as per Fakku request + dead
+            KSK, // Dead
             FAKKU, // Old Fakku; kept for retrocompatibility
             FAKKU2, // Dropped after Fakku decided to flag downloading accounts and IPs
             ASMHENTAI_COMICS, // Does not work directly

app/src/main/java/me/devsaki/hentoid/parsers/ParseHelper.java

@@ -563,4 +563,14 @@ public static String getImgSrc(@NonNull Element e) {
         if (result.isEmpty()) result = e.attr("data-cfsrc").trim(); // Cloudflare-served image
         return result;
     }
+
+    /**
+     * Generate the user agent corresponding to the given site
+     *
+     * @param site Site to generate the user-agent for
+     * @return User agent corresponding to the given site
+     */
+    public static String getUserAgent(Site site) {
+        return HttpHelper.getUserAgent(site.useMobileAgent(), site.useHentoidAgent(), site.useWebviewAgent());
+    }
 }

app/src/main/java/me/devsaki/hentoid/parsers/content/PixivContent.java

@@ -15,6 +15,7 @@
 import me.devsaki.hentoid.json.sources.PixivIllustMetadata;
 import me.devsaki.hentoid.json.sources.PixivSeriesMetadata;
 import me.devsaki.hentoid.json.sources.PixivUserMetadata;
+import me.devsaki.hentoid.parsers.ParseHelper;
 import me.devsaki.hentoid.retrofit.sources.PixivServer;
 import me.devsaki.hentoid.util.StringHelper;
 import me.devsaki.hentoid.util.network.HttpHelper;
@@ -54,25 +55,26 @@ public Content update(@NonNull final Content content, @Nonnull String url, boole
                         Site.PIXIV.useMobileAgent(), Site.PIXIV.useHentoidAgent(), Site.PIXIV.useWebviewAgent()
                 );
 
+                String userAgent = ParseHelper.getUserAgent(Site.PIXIV);
+                String acceptAll = "*/*";
+
                 switch (entity) {
-                    case "artworks":
-                    case "illust":
-                        PixivIllustMetadata metadata = PixivServer.api.getIllustMetadata(id, cookieStr).execute().body();
+                    case "artworks", "illust" -> {
+                        PixivIllustMetadata metadata = PixivServer.api.getIllustMetadata(id, cookieStr, acceptAll, userAgent).execute().body();
                         if (metadata != null) return metadata.update(content, url, updateImages);
-                        break;
-                    case "series_content":
-                    case "series":
-                        PixivSeriesMetadata seriesData = PixivServer.api.getSeriesMetadata(id, cookieStr).execute().body();
+                    }
+                    case "series_content", "series" -> {
+                        PixivSeriesMetadata seriesData = PixivServer.api.getSeriesMetadata(id, cookieStr, acceptAll, userAgent).execute().body();
                         if (seriesData != null)
                             return seriesData.update(content, url, updateImages);
-                        break;
-                    case "user":
-                    case "users":
-                        PixivUserMetadata userData = PixivServer.api.getUserMetadata(id, cookieStr).execute().body();
+                    }
+                    case "user", "users" -> {
+                        PixivUserMetadata userData = PixivServer.api.getUserMetadata(id, cookieStr, acceptAll, userAgent).execute().body();
                         if (userData != null) return userData.update(content, url, updateImages);
-                        break;
-                    default:
-                        // Nothing specific
+                    }
+                    default -> {
+                    }
+                    // Nothing specific
                 }
             }
         } catch (IOException e) {

app/src/main/java/me/devsaki/hentoid/parsers/images/PixivParser.java

@@ -67,15 +67,18 @@ private List<ImageFile> getPages(@NonNull Content onlineContent, @Nullable Conte
                     onlineContent.getGalleryUrl(), null,
                     useMobileAgent, useHentoidAgent, useWebviewAgent);
 
+            String userAgent = ParseHelper.getUserAgent(Site.PIXIV);
+            String acceptAll = "*/*";
+
             // API calls seem to be protected against request spam; 2 is arbitrary
             DownloadRateLimiter.INSTANCE.setRateLimit(2);
 
             if (onlineContent.getUrl().contains("/series/"))
-                return parseSeries(onlineContent, storedContent, cookieStr);
+                return parseSeries(onlineContent, storedContent, cookieStr, acceptAll, userAgent);
             else if (onlineContent.getUrl().contains("/artworks/"))
-                return parseIllust(onlineContent, cookieStr);
+                return parseIllust(onlineContent, cookieStr, acceptAll, userAgent);
             else if (onlineContent.getUrl().contains("users/"))
-                return parseUser(onlineContent, storedContent, cookieStr);
+                return parseUser(onlineContent, storedContent, cookieStr, acceptAll, userAgent);
         } catch (Exception e) {
             Timber.d(e);
             throw new EmptyResultException(StringHelper.protect(e.getMessage()));
@@ -84,9 +87,13 @@ else if (onlineContent.getUrl().contains("users/"))
         return Collections.emptyList();
     }
 
-    private List<ImageFile> parseIllust(@NonNull Content content, @NonNull String cookieStr) throws Exception {
+    private List<ImageFile> parseIllust(
+            @NonNull Content content,
+            @NonNull String cookieStr,
+            @NonNull String acceptAll,
+            @NonNull String userAgent) throws Exception {
         DownloadRateLimiter.INSTANCE.take();
-        PixivIllustPagesMetadata galleryMetadata = PixivServer.api.getIllustPages(content.getUniqueSiteId(), cookieStr).execute().body();
+        PixivIllustPagesMetadata galleryMetadata = PixivServer.api.getIllustPages(content.getUniqueSiteId(), cookieStr, acceptAll, userAgent).execute().body();
         if (null == galleryMetadata || galleryMetadata.isError()) {
             String message = "";
             if (galleryMetadata != null) message = galleryMetadata.getMessage();
@@ -95,7 +102,12 @@ private List<ImageFile> parseIllust(@NonNull Content content, @NonNull String co
         return ParseHelper.urlsToImageFiles(galleryMetadata.getPageUrls(), content.getCoverImageUrl(), StatusContent.SAVED);
     }
 
-    private List<ImageFile> parseSeries(@NonNull Content onlineContent, @Nullable Content storedContent, @NonNull String cookieStr) throws Exception {
+    private List<ImageFile> parseSeries(
+            @NonNull Content onlineContent,
+            @Nullable Content storedContent,
+            @NonNull String cookieStr,
+            @NonNull String acceptAll,
+            @NonNull String userAgent) throws Exception {
         String[] seriesIdParts = onlineContent.getUniqueSiteId().split("/");
         String seriesId = seriesIdParts[seriesIdParts.length - 1];
         if (seriesId.contains("?")) {
@@ -114,7 +126,7 @@ private List<ImageFile> parseSeries(@NonNull Content onlineContent, @Nullable Co
             if (processHalted.get()) break;
             int chaptersToRead = Math.min(nbChapters - chapters.size(), MAX_QUERY_WINDOW);
             DownloadRateLimiter.INSTANCE.take();
-            PixivSeriesIllustMetadata seriesContentMetadata = PixivServer.api.getSeriesIllusts(seriesId, chaptersToRead, chapters.size(), cookieStr).execute().body();
+            PixivSeriesIllustMetadata seriesContentMetadata = PixivServer.api.getSeriesIllusts(seriesId, chaptersToRead, chapters.size(), cookieStr, acceptAll, userAgent).execute().body();
             if (null == seriesContentMetadata || seriesContentMetadata.isError()) {
                 String message = "Unreachable series illust";
                 if (seriesContentMetadata != null)
@@ -150,7 +162,7 @@ private List<ImageFile> parseSeries(@NonNull Content onlineContent, @Nullable Co
         Set<Attribute> attrs = new HashSet<>();
         for (Chapter ch : extraChapters) {
             DownloadRateLimiter.INSTANCE.take();
-            PixivIllustMetadata illustMetadata = PixivServer.api.getIllustMetadata(ch.getUniqueId(), cookieStr).execute().body();
+            PixivIllustMetadata illustMetadata = PixivServer.api.getIllustMetadata(ch.getUniqueId(), cookieStr, acceptAll, userAgent).execute().body();
             if (null == illustMetadata || illustMetadata.isError()) {
                 String message = "Unreachable illust";
                 if (illustMetadata != null)
@@ -180,7 +192,12 @@ private List<ImageFile> parseSeries(@NonNull Content onlineContent, @Nullable Co
         return result;
     }
 
-    private List<ImageFile> parseUser(@NonNull Content onlineContent, @Nullable Content storedContent, @NonNull String cookieStr) throws Exception {
+    private List<ImageFile> parseUser(
+            @NonNull Content onlineContent,
+            @Nullable Content storedContent,
+            @NonNull String cookieStr,
+            @NonNull String acceptAll,
+            @NonNull String userAgent) throws Exception {
         String[] userIdParts = onlineContent.getUniqueSiteId().split("/");
         String userId = userIdParts[userIdParts.length - 1];
         if (userId.contains("?")) {
@@ -190,10 +207,10 @@ private List<ImageFile> parseUser(@NonNull Content onlineContent, @Nullable Cont
         // Retrieve the list of Illusts IDs (=chapters)
         int waited = 0;
         DownloadRateLimiter.INSTANCE.take();
-        Response<PixivUserIllustMetadata> userIllustResp = PixivServer.api.getUserIllusts(userId, cookieStr).execute();
+        Response<PixivUserIllustMetadata> userIllustResp = PixivServer.api.getUserIllusts(userId, cookieStr, acceptAll, userAgent).execute();
         if (HttpHelper.waitBlocking429(userIllustResp, Preferences.getHttp429DefaultDelaySecs() * 1000)) {
             waited++;
-            userIllustResp = PixivServer.api.getUserIllusts(userId, cookieStr).execute();
+            userIllustResp = PixivServer.api.getUserIllusts(userId, cookieStr, acceptAll, userAgent).execute();
         }
 
         if (userIllustResp.code() >= 400)
@@ -232,10 +249,10 @@ private List<ImageFile> parseUser(@NonNull Content onlineContent, @Nullable Cont
         for (String illustId : illustIds) {
             waited = 0;
             DownloadRateLimiter.INSTANCE.take();
-            Response<PixivIllustMetadata> illustResp = PixivServer.api.getIllustMetadata(illustId, cookieStr).execute();
+            Response<PixivIllustMetadata> illustResp = PixivServer.api.getIllustMetadata(illustId, cookieStr, acceptAll, userAgent).execute();
             while (HttpHelper.waitBlocking429(illustResp, Preferences.getHttp429DefaultDelaySecs() * 1000) && waited < 2) {
                 waited++;
-                illustResp = PixivServer.api.getIllustMetadata(illustId, cookieStr).execute();
+                illustResp = PixivServer.api.getIllustMetadata(illustId, cookieStr, acceptAll, userAgent).execute();
             }
 
             if (illustResp.code() >= 400)

app/src/main/java/me/devsaki/hentoid/retrofit/sources/PixivServer.kt

@@ -32,39 +32,51 @@ object PixivServer {
         @GET("touch/ajax/illust/details")
         fun getIllustMetadata(
             @Query("illust_id") id: String,
-            @Header("cookie") cookies: String
+            @Header("cookie") cookies: String,
+            @Header("accept") accept: String,
+            @Header("user-agent") userAgent: String
         ): Call<PixivIllustMetadata>
 
         @GET("ajax/illust/{id}/pages")
         fun getIllustPages(
             @Query("id") id: String,
-            @Header("cookie") cookies: String
+            @Header("cookie") cookies: String,
+            @Header("accept") accept: String,
+            @Header("user-agent") userAgent: String
         ): Call<PixivIllustPagesMetadata>
 
         @GET("touch/ajax/illust/series/{id}")
         fun getSeriesMetadata(
             @Path("id") id: String,
-            @Header("cookie") cookies: String
+            @Header("cookie") cookies: String,
+            @Header("accept") accept: String,
+            @Header("user-agent") userAgent: String
         ): Call<PixivSeriesMetadata>
 
         @GET("touch/ajax/illust/series_content/{id}")
         fun getSeriesIllusts(
             @Path("id") id: String,
             @Query("limit") limit: Int,
             @Query("last_order") lastorder: Int,
-            @Header("cookie") cookies: String
+            @Header("cookie") cookies: String,
+            @Header("accept") accept: String,
+            @Header("user-agent") userAgent: String
         ): Call<PixivSeriesIllustMetadata>
 
         @GET("touch/ajax/illust/user_illusts")
         fun getUserIllusts(
             @Query("user_id") id: String,
-            @Header("cookie") cookies: String
+            @Header("cookie") cookies: String,
+            @Header("accept") accept: String,
+            @Header("user-agent") userAgent: String
         ): Call<PixivUserIllustMetadata>
 
         @GET("touch/ajax/user/details")
         fun getUserMetadata(
             @Query("id") id: String,
-            @Header("cookie") cookies: String
+            @Header("cookie") cookies: String,
+            @Header("accept") accept: String,
+            @Header("user-agent") userAgent: String
         ): Call<PixivUserMetadata>
     }
 }

app/src/main/java/me/devsaki/hentoid/util/network/HttpHelper.java

@@ -48,7 +48,7 @@ public class HttpHelper {
     public static final String HEADER_COOKIE_KEY = "cookie";
     public static final String HEADER_REFERER_KEY = "referer";
     public static final String HEADER_CONTENT_TYPE = "Content-Type";
-    public static final String HEADER_USER_AGENT = "User-Agent";
+    public static final String HEADER_USER_AGENT = "user-agent";
 
     public static final String POST_MIME_TYPE = "application/x-www-form-urlencoded";
 
@@ -197,11 +197,23 @@ private static Request.Builder buildRequest(@NonNull String url, @Nullable List<
                 if (header.second != null)
                     requestBuilder.addHeader(header.first, header.second);
 
-        requestBuilder.header(HEADER_USER_AGENT, useMobileAgent ? getMobileUserAgent(useHentoidAgent, useWebviewAgent) : getDesktopUserAgent(useHentoidAgent, useWebviewAgent));
+        requestBuilder.header(HEADER_USER_AGENT, getUserAgent(useMobileAgent, useHentoidAgent, useWebviewAgent));
 
         return requestBuilder;
     }
 
+    /**
+     * Generate the user agent corresponding to the given parameters
+     *
+     * @param useMobileAgent  True to use the mobile User-Agent; false to use the desktop User-Agent
+     * @param useHentoidAgent True to use the Hentoid User-Agent; false to use a neutral User-Agent
+     * @param useWebviewAgent True to reveal the use of a webview through the User-Agent; false to use a neutral User-Agent
+     * @return User agent corresponding to the given parameters
+     */
+    public static String getUserAgent(boolean useMobileAgent, boolean useHentoidAgent, boolean useWebviewAgent) {
+        return useMobileAgent ? getMobileUserAgent(useHentoidAgent, useWebviewAgent) : getDesktopUserAgent(useHentoidAgent, useWebviewAgent);
+    }
+
     /**
      * Convert the given OkHttp {@link Response} into a {@link WebResourceResponse}, using the data from the given InputStream
      *
@@ -785,4 +797,4 @@ public void setQuery(String query) {
             this.query = query;
         }
     }
-}
+}