Support custom endpoints, check for non-2xx responses

README.md

@@ -22,9 +22,14 @@ go build
 ```
 
 ## Usage
-Fetch new (`diff-only`) assets for version `1.600.0` into `./dump` directory and extract them with `2` spaces indentation into `./output`:
+Fetch new (`diff-only`) raw assets for version `1.600.0` into `./dump` directory:
 ```sh
-wfax fetch --diff-only --version 1.600.0 ./dump && wfax extract --indent 2 ./dump ./output
+wfax fetch --diff-only --version 1.600.0 ./dump
+```
+
+Fetch raw assets from custom API and CDN endpoints (file URIs are also supported):
+```sh
+wfax fetch --custom-api file:///assets/asset_lists/en-android-full.json --custom-cdn file:///.cdn ./dump
 ```
 
 Fetch character comics (`--comics 1`) with `10` maximum concurrent requests into `./comics` directory:
@@ -42,9 +47,9 @@ Extract equipment image assets for eliyabot:
 wfax sprite --eliyabot ./dump ./output
 ```
 
-Pack extracted files in `./output` back into assets in `./dump`:
+Pack extracted files in `./output` into raw assets in `./repack`:
 ```sh
-wfax pack ./output ./dump
+wfax pack ./output ./repack
 ```
 
 For more detailed information, use `wfax help`.

cmd/fetch.go

@@ -14,6 +14,8 @@ var fetchDiff bool
 var fetchConcurrency int
 var fetchRegion string
 var fetchComics int
+var fetchCustomAPI string
+var fetchCustomCDN string
 
 var fetchCmd = &cobra.Command{
 	Use:   "fetch [target dir]",
@@ -24,6 +26,8 @@ var fetchCmd = &cobra.Command{
 			Version:     fetchVersion,
 			Workdir:     filepath.Clean(args[0]),
 			Concurrency: fetchConcurrency,
+			CustomAPI:   fetchCustomAPI,
+			CustomCDN:   fetchCustomCDN,
 		}
 		if fetchDiff {
 			config.Mode = wf.DiffAssets
@@ -71,4 +75,6 @@ func init() {
 	fetchCmd.Flags().IntVarP(&fetchConcurrency, "concurrency", "c", 5, "Maximum number of concurrent asset downloads")
 	fetchCmd.Flags().StringVarP(&fetchRegion, "region", "r", "jp", "Service region/language: jp, gl, th, kr, cn, tw")
 	fetchCmd.Flags().IntVarP(&fetchComics, "comics", "m", 0, "Fetch comics instead (1: character comics, 2: tutorial comics)")
+	fetchCmd.Flags().StringVarP(&fetchCustomAPI, "custom-api", "A", "", "Set custom API endpoint for asset metadata (file URIs also supported)")
+	fetchCmd.Flags().StringVarP(&fetchCustomCDN, "custom-cdn", "C", "", "Set custom CDN endpoint for assets (file URIs also supported)")
 }

pkg/wf/client.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/Jeffail/gabs/v2"
 	"github.com/blead/wfax/pkg/concurrency"
+	"github.com/hashicorp/go-cleanhttp"
 	retryablehttp "github.com/hashicorp/go-retryablehttp"
 	"github.com/tinylib/msgp/msgp"
 )
@@ -96,9 +97,9 @@ func getCDNAddress(region ServiceRegion) string {
 	}
 }
 
-func replaceCDNAddress(location string, region ServiceRegion) string {
-	if region == RegionGL || region == RegionTH || region == RegionKR {
-		return strings.ReplaceAll(location, "{$cdnAddress}", getCDNAddress(region))
+func replaceCDNAddress(location string, cdnAddress string) string {
+	if cdnAddress != "" {
+		return strings.ReplaceAll(location, "{$cdnAddress}", cdnAddress)
 	}
 	return location
 }
@@ -127,6 +128,8 @@ type ClientConfig struct {
 	Workdir     string
 	Concurrency int
 	Region      ServiceRegion
+	CustomAPI   string
+	CustomCDN   string
 }
 
 // DefaultClientConfig generates a default configuration.
@@ -137,6 +140,8 @@ func DefaultClientConfig() *ClientConfig {
 		Workdir:     "",
 		Concurrency: 5,
 		Region:      RegionJP,
+		CustomAPI:   "",
+		CustomCDN:   "",
 	}
 
 	return config
@@ -177,7 +182,11 @@ func NewClient(config *ClientConfig) (*Client, error) {
 		config.Concurrency = 5
 	}
 
+	transport := cleanhttp.DefaultPooledTransport()
+	transport.RegisterProtocol("file", http.NewFileTransport(http.Dir(".")))
+
 	client := retryablehttp.NewClient()
+	client.HTTPClient = &http.Client{Transport: transport}
 	client.Logger = log.Default()
 
 	return &Client{
@@ -212,12 +221,26 @@ func (client *Client) fetchMsgp(req *retryablehttp.Request) ([]byte, error) {
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("fetchMsqgp: non-2xx status code in response, status=%d, url=%s", resp.StatusCode, req.URL.String())
+	}
+
 	var output bytes.Buffer
-	_, err = msgp.CopyToJSON(&output, base64.NewDecoder(base64.StdEncoding, resp.Body))
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}
 
+	_, err = msgp.CopyToJSON(&output, base64.NewDecoder(base64.StdEncoding, bytes.NewReader(body)))
+	if err != nil {
+		// not base64, check if body is plain json
+		_, ok := err.(base64.CorruptInputError)
+		if ok && json.Valid(body) {
+			return body, nil
+		}
+		return nil, err
+	}
+
 	return output.Bytes(), nil
 }
 
@@ -232,31 +255,42 @@ func (client *Client) parseMetadata(json []byte, parseAssets bool) (string, []*a
 	if err != nil {
 		return "", nil, err
 	}
-	if !jsonParsed.ExistsP("data.info") {
+
+	// make outer data object optional to support asset list file in starpoint
+	if jsonParsed.ExistsP("data") {
+		jsonParsed = jsonParsed.Path("data")
+	}
+
+	if !jsonParsed.ExistsP("info") {
 		return "", []*assetMetadata{}, nil
 	}
 
-	version, ok := jsonParsed.Path("data.info.eventual_target_asset_version").Data().(string)
+	version, ok := jsonParsed.Path("info.eventual_target_asset_version").Data().(string)
 	if !ok {
 		return "", nil, fmt.Errorf("parseMetadata: unable to parse latest version number")
 	}
 
 	var assets []*assetMetadata
 
 	if parseAssets {
+		cdnAddress := client.config.CustomCDN
+		if cdnAddress == "" {
+			cdnAddress = getCDNAddress(client.config.Region)
+		}
+
 		if client.config.Mode == FullAssets {
-			for _, child := range jsonParsed.Path("data.full.archive").Children() {
+			for _, child := range jsonParsed.Path("full.archive").Children() {
 				assets = append(assets, &assetMetadata{
-					location: replaceCDNAddress(child.Path("location").Data().(string), client.config.Region),
+					location: replaceCDNAddress(child.Path("location").Data().(string), cdnAddress),
 					dest:     client.tmpDir,
 					sha256:   child.Path("sha256").Data().(string),
 				})
 			}
 		}
-		for _, group := range jsonParsed.Search("data", "diff", "*", "archive").Children() {
+		for _, group := range jsonParsed.Search("diff", "*", "archive").Children() {
 			for _, child := range group.Children() {
 				assets = append(assets, &assetMetadata{
-					location: replaceCDNAddress(child.Path("location").Data().(string), client.config.Region),
+					location: replaceCDNAddress(child.Path("location").Data().(string), cdnAddress),
 					dest:     client.tmpDir,
 					sha256:   child.Path("sha256").Data().(string),
 				})
@@ -280,6 +314,10 @@ func (client *Client) download(i *concurrency.Item[*assetMetadata, []string]) ([
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("download: non-2xx status code in response, status=%d, url=%s", resp.StatusCode, a.location)
+	}
+
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
@@ -296,7 +334,7 @@ func (client *Client) download(i *concurrency.Item[*assetMetadata, []string]) ([
 			return nil, err
 		}
 		if !bytes.Equal(expected, downloaded) {
-			return nil, fmt.Errorf("download: sha256 mismatch, expected: %x, downloaded: %x", expected, downloaded)
+			return nil, fmt.Errorf("download: sha256 mismatch, expected: %x, downloaded: %x, url: %s", expected, downloaded, a.location)
 		}
 	}
 
@@ -477,7 +515,12 @@ func (client *Client) buildComicListRequest(comicListReq *ComicListRequestBody)
 		return nil, err
 	}
 
-	req, err := retryablehttp.NewRequest("POST", getAPIEndpoint(client.config.Region, apiComicEndpoint), &body)
+	endpoint := client.config.CustomAPI
+	if endpoint == "" {
+		getAPIEndpoint(client.config.Region, apiComicEndpoint)
+	}
+
+	req, err := retryablehttp.NewRequest("POST", endpoint, &body)
 	if err != nil {
 		return nil, err
 	}
@@ -664,8 +707,13 @@ func (client *Client) FetchAssetsFromAPI(fetchComics int) error {
 		log.Println("[WARN] CN region is untested due to region block")
 	}
 
+	endpoint := client.config.CustomAPI
+	if endpoint == "" {
+		endpoint = getAPIEndpoint(client.config.Region, apiAssetEndpoint)
+	}
+
 	log.Println("[INFO] Fetching asset metadata, clientVersion=" + client.config.Version)
-	metadataReq, err := retryablehttp.NewRequest("GET", getAPIEndpoint(client.config.Region, apiAssetEndpoint), nil)
+	metadataReq, err := retryablehttp.NewRequest("GET", endpoint, nil)
 	if err != nil {
 		return err
 	}