From a58994baa27dc31d6613df36a6ae60b5836a06b3 Mon Sep 17 00:00:00 2001 From: Majd Alfhaily Date: Thu, 1 Oct 2026 21:47:08 +0200 Subject: [PATCH] fix: improve download reliability with kbsync --- cmd/list_versions.go | 2 +- internal/sap/machine/kbsync.go | 75 +++ internal/sap/machine/kbsync_test.go | 91 +++ internal/sap/machine/storeagent.go | 57 +- pkg/appstore/appstore.go | 3 + pkg/appstore/appstore_bag.go | 1 + pkg/appstore/appstore_download.go | 2 +- pkg/appstore/appstore_download_product.go | 61 +- .../appstore_download_product_test.go | 35 +- pkg/appstore/appstore_get_version_metadata.go | 2 +- pkg/appstore/appstore_kbsync.go | 151 +++++ pkg/appstore/appstore_kbsync_cache.go | 127 ++++ pkg/appstore/appstore_kbsync_cache_test.go | 157 +++++ pkg/appstore/appstore_kbsync_test.go | 632 ++++++++++++++++++ pkg/appstore/appstore_list_versions.go | 4 +- pkg/appstore/appstore_login.go | 3 + pkg/appstore/appstore_login_test.go | 3 +- .../appstore_macos_version_lookup_test.go | 7 +- pkg/appstore/appstore_revoke.go | 24 +- pkg/appstore/appstore_revoke_test.go | 19 + pkg/appstore/appstore_update_product_test.go | 23 +- pkg/appstore/authentication_retry_test.go | 54 +- pkg/http/client.go | 11 +- pkg/http/request.go | 4 + pkg/http/request_options_test.go | 75 +++ 25 files changed, 1530 insertions(+), 93 deletions(-) create mode 100644 internal/sap/machine/kbsync.go create mode 100644 internal/sap/machine/kbsync_test.go create mode 100644 pkg/appstore/appstore_kbsync.go create mode 100644 pkg/appstore/appstore_kbsync_cache.go create mode 100644 pkg/appstore/appstore_kbsync_cache_test.go create mode 100644 pkg/appstore/appstore_kbsync_test.go create mode 100644 pkg/http/request_options_test.go diff --git a/cmd/list_versions.go b/cmd/list_versions.go index edfc49e8..460ad8ea 100644 --- a/cmd/list_versions.go +++ b/cmd/list_versions.go @@ -63,7 +63,7 @@ func ListVersionsCmd() *cobra.Command { app = lookupResult.App } - out, err := dependencies.AppStore.ListVersions(appstore.ListVersionsInput{Account: acc, App: app, Platform: platform}) + out, err := dependencies.AppStore.ListVersions(appstore.ListVersionsInput{Context: cmd.Context(), Account: acc, App: app, Platform: platform}) if err != nil { return err } diff --git a/internal/sap/machine/kbsync.go b/internal/sap/machine/kbsync.go new file mode 100644 index 00000000..017562c6 --- /dev/null +++ b/internal/sap/machine/kbsync.go @@ -0,0 +1,75 @@ +package machine + +import ( + "context" + "errors" + "fmt" + + "github.com/majd/ipatool/v2/internal/sap/assets" +) + +// GenerateKBSync creates the account and hardware bound FairPlay data required +// by the bag's ent/download endpoint, without opening a decryption session. +func GenerateKBSync(ctx context.Context, bundle assets.Bundle, hardwareID []byte, dsid uint64) ([]byte, error) { + if ctx == nil { + return nil, errors.New("StoreAgent context is nil") + } + + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("generate kbsync: %w", err) + } + + if dsid == 0 { + return nil, errors.New("kbsync requires a nonzero account DSID") + } + + image, err := assets.LoadStoreAgent(ctx) + if err != nil { + return nil, fmt.Errorf("load Apple StoreAgent asset: %w", err) + } + + agent, globalContext, err := openStoreAgentGlobal(ctx, bundle, image, hardwareID) + if err != nil { + return nil, err + } + defer agent.Close() + + return agent.guest.generateKBSync(storeAgentKBSyncEntry, globalContext, dsid) +} + +func (m *Machine) generateKBSync(entry uint64, globalContext uint32, dsid uint64) ([]byte, error) { + m.beginCall() + defer m.clearScratch() + + pointerField, err := m.scratch(nil, 8) + if err != nil { + return nil, err + } + + // StoreAgent writes a uint32 length. Zero the upper half so consumeOutput + // can share the SAP buffer bounds checking and storage disposal. + lengthField, err := m.scratch(nil, 8) + if err != nil { + return nil, err + } + + status, err := m.invoke(entry, uint64(globalContext), dsid, 0, 1, pointerField, lengthField) + if err != nil { + return nil, fmt.Errorf("generate StoreAgent kbsync: %w", err) + } + + output, outputErr := m.consumeOutput(pointerField, lengthField) + if int32(status) != 0 { + return nil, errors.Join(fmt.Errorf("StoreAgent kbsync returned %d", int32(status)), outputErr) + } + + if outputErr != nil { + return nil, outputErr + } + + if len(output) == 0 { + return nil, errors.New("StoreAgent kbsync returned an empty buffer") + } + + return output, nil +} diff --git a/internal/sap/machine/kbsync_test.go b/internal/sap/machine/kbsync_test.go new file mode 100644 index 00000000..cbe44238 --- /dev/null +++ b/internal/sap/machine/kbsync_test.go @@ -0,0 +1,91 @@ +package machine + +import ( + "context" + "encoding/binary" + + "github.com/majd/ipatool/v2/internal/sap/assets" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("StoreAgent kbsync", func() { + It("pins the kbsync entry in the verified StoreAgent image", func() { + Expect(storeAgentKBSyncEntry).To(Equal(storeAgentBase + 0x0c93c0)) + }) + + It("rejects a canceled context before loading assets", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := GenerateKBSync(ctx, assets.Bundle{}, []byte{1}, 123) + Expect(err).To(MatchError(ContainSubstring(context.Canceled.Error()))) + }) + + It("rejects missing context or account identity before loading assets", func() { + var missingContext context.Context + _, err := GenerateKBSync(missingContext, assets.Bundle{}, nil, 123) + Expect(err).To(MatchError("StoreAgent context is nil")) + _, err = GenerateKBSync(context.Background(), assets.Bundle{}, nil, 0) + Expect(err).To(MatchError("kbsync requires a nonzero account DSID")) + }) + + DescribeTable("copies and disposes guest output with the StoreAgent ABI", + func(pointer, length uint64, status uint32, wantError string) { + guest := newGinkgoServiceMachine(shimOptions{}) + data := []byte{0, 4, 0, 1, 42} + Expect(guest.engine.MemWrite(heapBase, data)).To(Succeed()) + var disposed []uint64 + + dispose, err := guest.services.addFunction("test.kbsync.dispose", func() error { + address, err := guest.services.argument(0) + Expect(err).NotTo(HaveOccurred()) + disposed = append(disposed, address) + + return guest.services.setResult(0) + }) + Expect(err).NotTo(HaveOccurred()) + guest.entry.dispose = dispose + + entry, err := guest.services.addFunction("test.kbsync.generate", func() error { + for index, expected := range []uint64{0x1234, 0x123456789abcdef0, 0, 1} { + argument, err := guest.services.argument(index) + Expect(err).NotTo(HaveOccurred()) + Expect(argument).To(Equal(expected)) + } + + pointerField, err := guest.services.argument(4) + Expect(err).NotTo(HaveOccurred()) + Expect(guest.writeUint64(pointerField, pointer)).To(Succeed()) + lengthField, err := guest.services.argument(5) + Expect(err).NotTo(HaveOccurred()) + // Only the low 32 bits are written by the real entry point. + Expect(guest.engine.MemWrite(lengthField, binary.LittleEndian.AppendUint32(nil, uint32(length)))).To(Succeed()) + + return guest.services.setResult(uint64(status)) + }) + Expect(err).NotTo(HaveOccurred()) + + output, err := guest.generateKBSync(entry, 0x1234, 0x123456789abcdef0) + if wantError == "" { + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(Equal(data)) + } else { + Expect(err).To(MatchError(ContainSubstring(wantError))) + Expect(output).To(BeNil()) + } + + if pointer != 0 { + Expect(disposed).To(Equal([]uint64{pointer})) + } else { + Expect(disposed).To(BeEmpty()) + } + Expect(guest.scratchCursor).To(BeZero()) + Expect(guest.engine.MemRead(scratchBase, 32)).To(Equal(make([]byte, 32))) + }, + Entry("valid blob", heapBase, uint64(5), uint32(0), ""), + Entry("guest error still releases storage", heapBase, uint64(5), ^uint32(41), "kbsync returned -42"), + Entry("oversized blob", heapBase, maxOutputSize+1, uint32(0), "maximum is"), + Entry("null pointer", uint64(0), uint64(5), uint32(0), "null output pointer"), + Entry("empty blob", heapBase, uint64(0), uint32(0), "empty buffer"), + ) +}) diff --git a/internal/sap/machine/storeagent.go b/internal/sap/machine/storeagent.go index 6c17fd46..b1301863 100644 --- a/internal/sap/machine/storeagent.go +++ b/internal/sap/machine/storeagent.go @@ -13,6 +13,7 @@ import ( const ( storeAgentBase = uint64(0x00001000c0000000) storeAgentGlobalInit = storeAgentBase + 0x0c5fc0 + storeAgentKBSyncEntry = storeAgentBase + 0x0c93c0 storeAgentSessionInit = storeAgentBase + 0x0debd0 storeAgentDecryptEntry = storeAgentBase + 0x0ee700 storeAgentSessionClose = storeAgentBase + 0x1212d0 @@ -55,63 +56,69 @@ func OpenStoreAgent(ctx context.Context, bundle assets.Bundle, hardwareID, dpInf } func openStoreAgent(ctx context.Context, bundle assets.Bundle, image, hardwareID, dpInfo []byte) (*StoreAgent, error) { - if ctx == nil { - return nil, errors.New("StoreAgent context is nil") + if len(dpInfo) == 0 { + return nil, errors.New("StoreAgent dpInfo is empty") } - if err := ctx.Err(); err != nil { - return nil, fmt.Errorf("open StoreAgent runtime: %w", err) + agent, globalContext, err := openStoreAgentGlobal(ctx, bundle, image, hardwareID) + if err != nil { + return nil, err } - if len(dpInfo) == 0 { - return nil, errors.New("StoreAgent dpInfo is empty") + agent.session, err = agent.initializeSession(globalContext, dpInfo) + if err != nil { + _ = agent.Close() + + return nil, err + } + + return agent, nil +} + +func openStoreAgentGlobal(ctx context.Context, bundle assets.Bundle, image, hardwareID []byte) (*StoreAgent, uint32, error) { + if ctx == nil { + return nil, 0, errors.New("StoreAgent context is nil") + } + + if err := ctx.Err(); err != nil { + return nil, 0, fmt.Errorf("open StoreAgent runtime: %w", err) } if err := assets.VerifyStoreAgent(image); err != nil { - return nil, fmt.Errorf("verify Apple StoreAgent profile: %w", err) + return nil, 0, fmt.Errorf("verify Apple StoreAgent profile: %w", err) } hardware, err := hardwareBlock(hardwareID) if err != nil { - return nil, err + return nil, 0, err } - guest, _, err := openRuntime(ctx, bundle, runtimeOptions{ + guest, exports, err := openRuntime(ctx, bundle, runtimeOptions{ extraImages: []imageSpec{{name: "storeagent", data: image, base: storeAgentBase}}, shims: shimOptions{ zeroReturnAliases: storeAgentZeroReturnAliases, }, }) if err != nil { - return nil, fmt.Errorf("start StoreAgent runtime: %w", err) + return nil, 0, fmt.Errorf("start StoreAgent runtime: %w", err) } + guest.entry.dispose = exports["_jEHf8Xzsv8K"] + agent := &StoreAgent{ guest: guest, decryptEntry: storeAgentDecryptEntry, closeEntry: storeAgentSessionClose, } - complete := false - - defer func() { - if !complete { - _ = agent.Close() - } - }() globalContext, err := agent.initializeGlobal(hardware) if err != nil { - return nil, err - } + _ = agent.Close() - agent.session, err = agent.initializeSession(globalContext, dpInfo) - if err != nil { - return nil, err + return nil, 0, err } - complete = true - - return agent, nil + return agent, globalContext, nil } func (s *StoreAgent) initializeGlobal(hardware []byte) (uint32, error) { diff --git a/pkg/appstore/appstore.go b/pkg/appstore/appstore.go index cfc3b6b7..d5e2d036 100644 --- a/pkg/appstore/appstore.go +++ b/pkg/appstore/appstore.go @@ -43,6 +43,8 @@ type appstore struct { searchClient http.Client[searchResult] purchaseClient http.Client[purchaseResult] downloadClient http.Client[downloadResult] + kbsyncGenerator kbsyncGenerator + kbsyncCache kbsyncCache platformClient http.Client[platformVersionLookupResult] storefrontClient http.Client[[]byte] bagClient http.Client[bagResult] @@ -79,6 +81,7 @@ func NewAppStore(args Args) AppStore { searchClient: http.NewClient[searchResult](clientArgs), purchaseClient: http.NewClient[purchaseResult](clientArgs), downloadClient: http.NewClient[downloadResult](clientArgs), + kbsyncGenerator: defaultKBSyncGenerator, platformClient: http.NewClient[platformVersionLookupResult](clientArgs), storefrontClient: http.NewClient[[]byte](clientArgs), bagClient: http.NewClient[bagResult](http.Args{CookieJar: args.CookieJar, Timeout: http.DefaultAuthenticationTimeout}), diff --git a/pkg/appstore/appstore_bag.go b/pkg/appstore/appstore_bag.go index d4c4b640..d71f285d 100644 --- a/pkg/appstore/appstore_bag.go +++ b/pkg/appstore/appstore_bag.go @@ -77,6 +77,7 @@ type bagResult struct { type urlBag struct { AuthEndpoint string `plist:"authenticateAccount,omitempty"` + EntDownloadEndpoint string `plist:"volumeStoreDownloadProduct,omitempty"` RedownloadEndpoint string `plist:"redownloadProduct,omitempty"` UpdateEndpoint string `plist:"updateProduct,omitempty"` SAPSetupEndpoint string `plist:"sign-sap-setup,omitempty"` diff --git a/pkg/appstore/appstore_download.go b/pkg/appstore/appstore_download.go index 2c1e512b..ca813a25 100644 --- a/pkg/appstore/appstore_download.go +++ b/pkg/appstore/appstore_download.go @@ -63,7 +63,7 @@ func (t *appstore) Download(input DownloadInput) (DownloadOutput, error) { } } - res, resolvedPlatform, err := t.sendDownloadProduct(input.Account, input.App, guid, externalVersionID, input.Platform) + res, resolvedPlatform, err := t.sendDownloadProduct(input.Context, input.Account, input.App, guid, externalVersionID, input.Platform) if err != nil { return DownloadOutput{}, err } diff --git a/pkg/appstore/appstore_download_product.go b/pkg/appstore/appstore_download_product.go index 63786b16..9aef9ff7 100644 --- a/pkg/appstore/appstore_download_product.go +++ b/pkg/appstore/appstore_download_product.go @@ -1,6 +1,7 @@ package appstore import ( + "context" "errors" "fmt" gohttp "net/http" @@ -23,7 +24,15 @@ type downloadProductEndpoint struct { versionKey string } -func (t *appstore) sendDownloadProduct(acc Account, app App, guid, externalVersionID string, platform Platform) (http.Result[downloadResult], Platform, error) { +func (t *appstore) sendDownloadProduct(ctx context.Context, acc Account, app App, guid, externalVersionID string, platform Platform) (http.Result[downloadResult], Platform, error) { + if ctx == nil { + ctx = context.Background() + } + + if err := ctx.Err(); err != nil { + return http.Result[downloadResult]{}, platform, fmt.Errorf("download request canceled: %w", err) + } + // An unpinned request can return the iOS build of a universal app. // Select the Mac offer before accepting any download response. if externalVersionID == "" && platform == PlatformMacOS { @@ -35,6 +44,28 @@ func (t *appstore) sendDownloadProduct(acc Account, app App, guid, externalVersi } } + // Prefer ent/download when its generator and bag endpoint are available. + // Keep the fetched bag for the legacy chain if this attempt cannot serve + // the requested app; asset, network, and response failures can fall back. + var bag *urlBag + + if t.kbsyncGenerator != nil { + fetched, err := t.fetchURLBag(guid) + if err == nil { + bag = &fetched + if bag.EntDownloadEndpoint != "" { + res, resolvedPlatform, err := t.sendPreferredDownload(ctx, bag.EntDownloadEndpoint, acc, app, guid, externalVersionID, platform) + if err == nil { + return res, resolvedPlatform, nil + } + } + } + } + + if err := ctx.Err(); err != nil { + return http.Result[downloadResult]{}, platform, fmt.Errorf("download request canceled: %w", err) + } + volumeStore := t.volumeStoreEndpoint(acc) res, err := t.downloadClient.Send(t.downloadProductRequest(volumeStore, acc, app, guid, externalVersionID)) @@ -48,9 +79,13 @@ func (t *appstore) sendDownloadProduct(acc Account, app App, guid, externalVersi // Try redownload when volumeStore returns no items, either silently or // with the message-only "No Longer Available" response. - bag, err := t.fetchURLBag(guid) - if err != nil { - return res, platform, fmt.Errorf("failed to get bag for redownload fallback: %w", err) + if bag == nil { + fetched, err := t.fetchURLBag(guid) + if err != nil { + return res, platform, fmt.Errorf("failed to get bag for redownload fallback: %w", err) + } + + bag = &fetched } if bag.RedownloadEndpoint == "" { @@ -116,34 +151,38 @@ func (t *appstore) sendUpdateProduct(endpoint string, acc Account, app App, guid return res, fmt.Errorf("failed to send update request: %w", err) } + return res, validateVersionedDownloadResponse(res, app, externalVersionID, "update") +} + +func validateVersionedDownloadResponse(res http.Result[downloadResult], app App, externalVersionID, source string) error { if res.Data.FailureType != "" { - return res, nil + return nil } if res.Data.CustomerMessage != "" { - return res, NewErrorWithMetadata(fmt.Errorf("received update error: %s", res.Data.CustomerMessage), res) + return NewErrorWithMetadata(fmt.Errorf("received %s error: %s", source, res.Data.CustomerMessage), res) } if res.StatusCode != gohttp.StatusOK { - return res, fmt.Errorf("received unexpected update status code: %d", res.StatusCode) + return fmt.Errorf("received unexpected %s status code: %d", source, res.StatusCode) } if len(res.Data.Items) != 1 { - return res, errors.New("update response must contain exactly one item") + return fmt.Errorf("%s response must contain exactly one item", source) } metadata := res.Data.Items[0].Metadata if fmt.Sprint(metadata["itemId"]) != fmt.Sprint(app.ID) || fmt.Sprint(metadata["softwareVersionExternalIdentifier"]) != externalVersionID { - return res, errors.New("update response does not match the requested app or version") + return fmt.Errorf("%s response does not match the requested app or version", source) } bundleID, ok := metadata["softwareVersionBundleId"].(string) if !ok || bundleID == "" || (app.BundleID != "" && bundleID != app.BundleID) { - return res, errors.New("update response does not match the requested bundle identifier") + return fmt.Errorf("%s response does not match the requested bundle identifier", source) } - return res, nil + return nil } func newDownloadEndpoint(endpoint, path string) (downloadProductEndpoint, error) { diff --git a/pkg/appstore/appstore_download_product_test.go b/pkg/appstore/appstore_download_product_test.go index 6016aded..683eb652 100644 --- a/pkg/appstore/appstore_download_product_test.go +++ b/pkg/appstore/appstore_download_product_test.go @@ -1,6 +1,7 @@ package appstore import ( + "context" "errors" "fmt" gohttp "net/http" @@ -56,7 +57,7 @@ var _ = Describe("AppStore (Download Product)", func() { ctrl.Finish() }) - It("uses volumeStore as the primary endpoint", func() { + It("uses volumeStore when kbsync generation is unavailable", func() { expected := http.Result[downloadResult]{ StatusCode: gohttp.StatusOK, Data: downloadResult{ @@ -78,7 +79,7 @@ var _ = Describe("AppStore (Download Product)", func() { }). Return(expected, nil) - actual, platform, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + actual, platform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) Expect(platform).To(Equal(PlatformIPhone)) @@ -97,7 +98,7 @@ var _ = Describe("AppStore (Download Product)", func() { Send(gomock.Any()). Return(expected, nil) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) }) @@ -137,7 +138,7 @@ var _ = Describe("AppStore (Download Product)", func() { Return(expected, nil), ) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) }) @@ -152,7 +153,7 @@ var _ = Describe("AppStore (Download Product)", func() { Send(gomock.Any()). Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: validBagResult()}, nil) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) }) @@ -165,7 +166,7 @@ var _ = Describe("AppStore (Download Product)", func() { Send(gomock.Any()). Return(http.Result[bagResult]{}, errors.New("bag request failed")) - _, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).To(MatchError(ContainSubstring("failed to get bag for redownload fallback"))) }) @@ -185,7 +186,7 @@ var _ = Describe("AppStore (Download Product)", func() { Return(http.Result[downloadResult]{}, errors.New("redownload failed")), ) - _, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).To(MatchError(ContainSubstring("failed to send redownload request"))) }) @@ -197,7 +198,7 @@ var _ = Describe("AppStore (Download Product)", func() { } mockDownloadClient.EXPECT().Send(gomock.Any()).Return(expected, nil) - actual, resolvedPlatform, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, platform) + actual, resolvedPlatform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, platform) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) Expect(resolvedPlatform).To(Equal(platform)) @@ -265,7 +266,7 @@ var _ = Describe("AppStore (Download Product)", func() { Expect(payload).ToNot(HaveKey("pricingParameters")) }).Return(expected, nil), ) - actual, resolvedPlatform, err := store.sendDownloadProduct(account, app, testGUID, "", platform) + actual, resolvedPlatform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", platform) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) if platform == "" { @@ -288,7 +289,7 @@ var _ = Describe("AppStore (Download Product)", func() { Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("appExtVrsId", testVersionID)) }).Return(http.Result[downloadResult]{StatusCode: gohttp.StatusOK}, nil), ) - _, _, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformIPhone) Expect(err).ToNot(HaveOccurred()) }) @@ -302,7 +303,7 @@ var _ = Describe("AppStore (Download Product)", func() { Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("appExtVrsId", testVersionID)) }).Return(http.Result[downloadResult]{}, original), ) - _, _, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformIPhone) Expect(errors.Is(err, original)).To(BeTrue()) }, Entry("empty HTTP 500", fmt.Errorf("wrapped: %w", &http.UnexpectedResponseError{StatusCode: gohttp.StatusInternalServerError})), @@ -328,7 +329,7 @@ var _ = Describe("AppStore (Download Product)", func() { Expect(payload).To(HaveKeyWithValue("appExtVrsId", versionID)) } }).Return(expected, nil) - actual, resolvedPlatform, err := store.sendDownloadProduct(account, app, testGUID, versionID, platform) + actual, resolvedPlatform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, versionID, platform) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) Expect(resolvedPlatform).To(Equal(platform)) @@ -349,7 +350,7 @@ var _ = Describe("AppStore (Download Product)", func() { Return(http.Result[platformVersionLookupResult]{StatusCode: gohttp.StatusOK, Data: latestVersion}, nil), mockDownloadClient.EXPECT().Send(gomock.Any()).Return(expected, nil), ) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformIPhone) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformIPhone) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) }) @@ -358,7 +359,7 @@ var _ = Describe("AppStore (Download Product)", func() { lookupErr := errors.New("catalog unavailable") mockPlatformClient.EXPECT().Send(gomock.Any()).Return(http.Result[platformVersionLookupResult]{}, lookupErr) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformIPhone) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformIPhone) Expect(isEmptyDownloadProductResponse(actual)).To(BeTrue()) Expect(errors.Is(err, lookupErr)).To(BeTrue()) Expect(err).To(MatchError(ContainSubstring("failed to resolve latest iOS version for redownload"))) @@ -368,7 +369,7 @@ var _ = Describe("AppStore (Download Product)", func() { mockPlatformClient.EXPECT().Send(gomock.Any()). Return(http.Result[platformVersionLookupResult]{StatusCode: gohttp.StatusOK}, nil).Times(3) - _, _, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformIPhone) Expect(err).To(MatchError(ContainSubstring("platform version lookup returned no app"))) }) @@ -379,7 +380,7 @@ var _ = Describe("AppStore (Download Product)", func() { mockPlatformClient.EXPECT().Send(gomock.Any()). Return(http.Result[platformVersionLookupResult]{StatusCode: gohttp.StatusOK, Data: latestVersion}, nil) - _, _, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformIPhone) Expect(err).To(MatchError(ContainSubstring("platform version lookup returned no external version id"))) }) @@ -390,7 +391,7 @@ var _ = Describe("AppStore (Download Product)", func() { Return(http.Result[platformVersionLookupResult]{StatusCode: gohttp.StatusOK, Data: latestVersion}, nil), mockDownloadClient.EXPECT().Send(gomock.Any()).Return(expected, nil), ) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformIPhone) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformIPhone) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) }) diff --git a/pkg/appstore/appstore_get_version_metadata.go b/pkg/appstore/appstore_get_version_metadata.go index 44229961..964c0024 100644 --- a/pkg/appstore/appstore_get_version_metadata.go +++ b/pkg/appstore/appstore_get_version_metadata.go @@ -40,7 +40,7 @@ func (t *appstore) GetVersionMetadata(input GetVersionMetadataInput) (GetVersion guid := strings.ReplaceAll(strings.ToUpper(macAddr), ":", "") - res, _, err := t.sendDownloadProduct(input.Account, input.App, guid, input.VersionID, platform) + res, _, err := t.sendDownloadProduct(input.Context, input.Account, input.App, guid, input.VersionID, platform) if err != nil { return GetVersionMetadataOutput{}, err } diff --git a/pkg/appstore/appstore_kbsync.go b/pkg/appstore/appstore_kbsync.go new file mode 100644 index 00000000..ef6779ba --- /dev/null +++ b/pkg/appstore/appstore_kbsync.go @@ -0,0 +1,151 @@ +package appstore + +import ( + "context" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strconv" + "time" + + "github.com/majd/ipatool/v2/internal/sap/assets" + "github.com/majd/ipatool/v2/internal/sap/machine" + "github.com/majd/ipatool/v2/pkg/http" +) + +const entDownloadPath = "/WebObjects/DownloadDispatch.woa/wa/ent/download" + +type kbsyncGenerator func(context.Context, []byte, uint64) ([]byte, error) + +func defaultKBSyncGenerator(ctx context.Context, hardwareID []byte, dsid uint64) ([]byte, error) { + bundle, err := assets.Load(ctx) + if err != nil { + return nil, fmt.Errorf("load SAP assets for kbsync: %w", err) + } + + data, err := machine.GenerateKBSync(ctx, bundle, hardwareID, dsid) + if err != nil { + return nil, fmt.Errorf("generate kbsync: %w", err) + } + + return data, nil +} + +func (t *appstore) sendPreferredDownload(ctx context.Context, endpointURL string, acc Account, app App, guid, versionID string, platform Platform) (http.Result[downloadResult], Platform, error) { + endpoint, err := newDownloadEndpoint(endpointURL, entDownloadPath) + if err != nil { + return http.Result[downloadResult]{}, platform, err + } + + if platform == "" { + platform = PlatformIPhone + } + + if versionID == "" { + versionID, err = t.lookupLatestExternalVersionID(acc, app, platform) + if err != nil { + return http.Result[downloadResult]{}, platform, fmt.Errorf("failed to resolve platform version for ent/download: %w", err) + } + } + + res, err := t.sendEntDownload(ctx, endpoint, acc, app, guid, versionID) + + return res, platform, err +} + +func (t *appstore) sendEntDownload(ctx context.Context, endpoint downloadProductEndpoint, acc Account, app App, guid, versionID string) (http.Result[downloadResult], error) { + hardwareID, err := hex.DecodeString(guid) + if err != nil || len(hardwareID) != 6 { + return http.Result[downloadResult]{}, errors.New("ent/download requires a six-byte device GUID") + } + + dsid, err := strconv.ParseUint(acc.DirectoryServicesID, 10, 64) + if err != nil || dsid == 0 { + return http.Result[downloadResult]{}, errors.New("ent/download requires a nonzero numeric account DSID") + } + + if ctx == nil { + ctx = context.Background() + } + + // Asset loading and guest execution also honor this deadline. Each guest + // call retains the emulator's existing one-minute execution bound. + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + if err := ctx.Err(); err != nil { + return http.Result[downloadResult]{}, fmt.Errorf("ent/download canceled: %w", err) + } + + if cached := t.cachedKBSync(dsid, guid); cached != "" { + res, err := t.sendEntDownloadRequest(ctx, endpoint, acc, app, guid, versionID, hardwareID, cached) + if err == nil { + return res, nil + } + + if ctx.Err() != nil { + return res, fmt.Errorf("ent/download canceled: %w", ctx.Err()) + } + + // Retry a rejected cached blob once, bypassing the cache even if its + // persistent entry could not be removed. Fresh blobs are never retried. + t.invalidateKBSync(dsid, guid, cached) + } + + blob, err := t.generateKBSync(ctx, hardwareID, dsid) + if err != nil { + return http.Result[downloadResult]{}, err + } + + res, err := t.sendEntDownloadRequest(ctx, endpoint, acc, app, guid, versionID, hardwareID, blob) + if err == nil { + t.storeKBSync(dsid, guid, blob) + } + + return res, err +} + +func (t *appstore) sendEntDownloadRequest(ctx context.Context, endpoint downloadProductEndpoint, acc Account, app App, guid, versionID string, hardwareID []byte, blob string) (http.Result[downloadResult], error) { + serial := append([]byte{0x54, 0xc8, 0xb0, 0xa9, 0x88}, hardwareID[2:]...) + request := t.downloadProductRequest(endpoint, acc, app, guid, "") + + requestContext, cancelRequest := context.WithTimeout(ctx, http.DefaultAuthenticationTimeout) + defer cancelRequest() + + request.Context = requestContext + + // X-Token must not be forwarded to redirect destinations. + request.NoRedirects = true + request.Headers["Content-Type"] = "application/x-www-form-urlencoded; charset=utf-8" + request.Headers["User-Agent"] = "Configurator/2.18 (Macintosh; OS X 15.3.2; 24D81) AppleWebKit/0620.2.4.11.6" + request.Headers["X-Apple-Store-Front"] = acc.StoreFront + request.Headers["X-Token"] = acc.PasswordToken + request.Payload = &http.XMLPayload{Content: map[string]interface{}{ + "creditDisplay": "", + "guid": guid, + "kbsync": blob, + "salableAdamId": strconv.FormatInt(app.ID, 10), + "serialNumber": base64.StdEncoding.EncodeToString(serial), + "externalVersionId": versionID, + }} + + res, err := t.downloadClient.Send(request) + if err != nil { + return res, fmt.Errorf("send ent/download request: %w", err) + } + + if res.Data.FailureType != "" { + return res, NewErrorWithMetadata(fmt.Errorf("received ent/download error: %s", res.Data.FailureType), res) + } + + if err := validateVersionedDownloadResponse(res, app, versionID, "ent/download"); err != nil { + return res, err + } + + if res.Data.Items[0].URL == "" { + return res, errors.New("ent/download response has no package URL") + } + + return res, nil +} diff --git a/pkg/appstore/appstore_kbsync_cache.go b/pkg/appstore/appstore_kbsync_cache.go new file mode 100644 index 00000000..c5aed2e3 --- /dev/null +++ b/pkg/appstore/appstore_kbsync_cache.go @@ -0,0 +1,127 @@ +package appstore + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "sync" +) + +// Version the key when the pinned StoreAgent profile or cache format changes. +const kbsyncCacheKey = "kbsync-v1" + +type kbsyncCacheEntry struct { + DSID uint64 `json:"dsid"` + GUID string `json:"guid"` + Data string `json:"data"` +} + +// Cache a single account/device pair, separately from Account so the blob is +// neither exposed in account output nor overwritten by a token refresh. +type kbsyncCache struct { + mu sync.Mutex + entry kbsyncCacheEntry +} + +func (t *appstore) generateKBSync(ctx context.Context, hardwareID []byte, dsid uint64) (string, error) { + if err := ctx.Err(); err != nil { + return "", fmt.Errorf("generate kbsync: %w", err) + } + + if t.kbsyncGenerator == nil { + return "", errors.New("kbsync generator is unavailable") + } + + data, err := t.kbsyncGenerator(ctx, hardwareID, dsid) + if err != nil { + return "", fmt.Errorf("generate download kbsync: %w", err) + } + + if err := ctx.Err(); err != nil { + return "", fmt.Errorf("generate kbsync: %w", err) + } + + if len(data) == 0 { + return "", errors.New("generated download kbsync is empty") + } + + return base64.StdEncoding.EncodeToString(data), nil +} + +// Store only blobs that have served a valid ent/download response. +func (t *appstore) storeKBSync(dsid uint64, guid, data string) { + entry := kbsyncCacheEntry{DSID: dsid, GUID: guid, Data: data} + + t.kbsyncCache.mu.Lock() + defer t.kbsyncCache.mu.Unlock() + + t.kbsyncCache.entry = entry + if t.keychain != nil { + // Cache storage is best effort: an unavailable keychain must not turn + // successful generation into a failed download. + if encoded, err := json.Marshal(entry); err == nil { + _ = t.keychain.Set(kbsyncCacheKey, encoded) + } + } +} + +func (t *appstore) cachedKBSync(dsid uint64, guid string) string { + t.kbsyncCache.mu.Lock() + defer t.kbsyncCache.mu.Unlock() + + entry := t.kbsyncCache.entry + if entry.DSID == dsid && entry.GUID == guid && entry.Data != "" { + return entry.Data + } + + if t.keychain == nil { + return "" + } + + encoded, err := t.keychain.Get(kbsyncCacheKey) + if err != nil { + return "" + } + + entry = kbsyncCacheEntry{} + if json.Unmarshal(encoded, &entry) != nil || entry.DSID != dsid || entry.GUID != guid { + return "" + } + + data, err := base64.StdEncoding.DecodeString(entry.Data) + if err != nil || len(data) == 0 { + return "" + } + + t.kbsyncCache.entry = entry + + return entry.Data +} + +func (t *appstore) invalidateKBSync(dsid uint64, guid, data string) { + t.kbsyncCache.mu.Lock() + defer t.kbsyncCache.mu.Unlock() + + rejected := kbsyncCacheEntry{DSID: dsid, GUID: guid, Data: data} + if t.kbsyncCache.entry == rejected { + t.kbsyncCache.entry = kbsyncCacheEntry{} + } + + if t.keychain == nil { + return + } + + // A concurrent request may have cached a different account or a newer + // blob. Only remove the exact entry used by the failed request. + encoded, err := t.keychain.Get(kbsyncCacheKey) + if err != nil { + return + } + + var entry kbsyncCacheEntry + if json.Unmarshal(encoded, &entry) == nil && entry == rejected { + _ = t.keychain.Remove(kbsyncCacheKey) + } +} diff --git a/pkg/appstore/appstore_kbsync_cache_test.go b/pkg/appstore/appstore_kbsync_cache_test.go new file mode 100644 index 00000000..2ae84c29 --- /dev/null +++ b/pkg/appstore/appstore_kbsync_cache_test.go @@ -0,0 +1,157 @@ +package appstore + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + + "github.com/byteness/keyring" + "github.com/majd/ipatool/v2/pkg/keychain" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" +) + +var _ = Describe("Persistent kbsync cache", func() { + const guid = "001122334455" + const dsid = uint64(123456789) + var ( + chain *keychain.MockKeychain + stored []byte + readErr, writeErr error + removed int + blob string + newStore func() *appstore + ) + BeforeEach(func() { + chain = keychain.NewMockKeychain(gomock.NewController(GinkgoT())) + stored = nil + readErr, writeErr = nil, nil + removed = 0 + blob = base64.StdEncoding.EncodeToString([]byte("working blob")) + chain.EXPECT().Get(kbsyncCacheKey).DoAndReturn(func(string) ([]byte, error) { + if readErr != nil { + return nil, readErr + } + if stored == nil { + return nil, keyring.ErrKeyNotFound + } + + return bytes.Clone(stored), nil + }).AnyTimes() + chain.EXPECT().Set(kbsyncCacheKey, gomock.Any()).DoAndReturn(func(_ string, data []byte) error { + if writeErr != nil { + return writeErr + } + stored = bytes.Clone(data) + + return nil + }).AnyTimes() + chain.EXPECT().Remove(kbsyncCacheKey).DoAndReturn(func(string) error { + removed++ + stored = nil + + return nil + }).AnyTimes() + newStore = func() *appstore { return &appstore{keychain: chain} } + }) + + It("reuses a keychain blob in a fresh client with the same account and device", func() { + newStore().storeKBSync(dsid, guid, blob) + var entry kbsyncCacheEntry + Expect(json.Unmarshal(stored, &entry)).To(Succeed()) + Expect(entry).To(Equal(kbsyncCacheEntry{DSID: dsid, GUID: guid, Data: blob})) + Expect(newStore().cachedKBSync(dsid, guid)).To(Equal(blob)) + }) + + DescribeTable("misses for a different account or device", func(nextDSID uint64, nextGUID string) { + store := newStore() + store.storeKBSync(dsid, guid, blob) + Expect(store.cachedKBSync(nextDSID, nextGUID)).To(BeEmpty()) + Expect(newStore().cachedKBSync(nextDSID, nextGUID)).To(BeEmpty()) + }, Entry("account changed", dsid+1, guid), Entry("device changed", dsid, "001122334466")) + + DescribeTable("ignores an unusable persistent entry", func(record string) { + stored = []byte(record) + Expect(newStore().cachedKBSync(dsid, guid)).To(BeEmpty()) + }, + Entry("malformed JSON", "{"), Entry("missing identity", `{"data":"YmxvYg=="}`), + Entry("invalid base64", `{"dsid":123456789,"guid":"001122334455","data":"!"}`), + Entry("empty data", `{"dsid":123456789,"guid":"001122334455","data":""}`), + ) + + It("does not borrow data from another account when a persistent entry is incomplete", func() { + store := newStore() + store.storeKBSync(dsid, guid, blob) + stored = []byte(`{"dsid":987654321,"guid":"001122334455"}`) + Expect(store.cachedKBSync(987654321, guid)).To(BeEmpty()) + }) + + It("keeps the memory cache usable when keychain reads or writes fail", func() { + readErr = errors.New("cache read failed") + writeErr = errors.New("cache write failed") + store := newStore() + Expect(store.cachedKBSync(dsid, guid)).To(BeEmpty()) + store.storeKBSync(dsid, guid, blob) + Expect(store.cachedKBSync(dsid, guid)).To(Equal(blob)) + Expect(stored).To(BeNil()) + }) + + It("does not cache a generated blob before it has been used", func() { + store := newStore() + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { return []byte("new blob"), nil } + generated, err := store.generateKBSync(context.Background(), nil, dsid) + Expect(err).NotTo(HaveOccurred()) + Expect(generated).NotTo(BeEmpty()) + Expect(store.cachedKBSync(dsid, guid)).To(BeEmpty()) + Expect(stored).To(BeNil()) + }) + + It("rejects failed generation and empty blobs", func() { + store := newStore() + failure := errors.New("guest failed") + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { return nil, failure } + _, err := store.generateKBSync(context.Background(), nil, dsid) + Expect(errors.Is(err, failure)).To(BeTrue()) + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { return nil, nil } + _, err = store.generateKBSync(context.Background(), nil, dsid) + Expect(err).To(HaveOccurred()) + Expect(stored).To(BeNil()) + }) + + It("removes a rejected blob from both caches", func() { + store := newStore() + store.storeKBSync(dsid, guid, blob) + store.invalidateKBSync(dsid, guid, blob) + Expect(removed).To(Equal(1)) + Expect(stored).To(BeNil()) + Expect(store.cachedKBSync(dsid, guid)).To(BeEmpty()) + }) + + It("clears persistent and in-memory blobs when revoking the account", func() { + store := newStore() + store.storeKBSync(dsid, guid, blob) + chain.EXPECT().Remove("account").Return(nil) + Expect(store.Revoke()).To(Succeed()) + Expect(stored).To(BeNil()) + Expect(removed).To(Equal(1)) + Expect(store.cachedKBSync(dsid, guid)).To(BeEmpty()) + }) + + It("does not evict a different account or a newer persisted blob", func() { + first := newStore() + first.storeKBSync(dsid, guid, blob) + newStore().storeKBSync(dsid+1, guid, blob) + first.invalidateKBSync(dsid, guid, blob) + Expect(removed).To(BeZero()) + Expect(newStore().cachedKBSync(dsid+1, guid)).To(Equal(blob)) + + newer := base64.StdEncoding.EncodeToString([]byte("newer")) + newStore().storeKBSync(dsid, guid, newer) + first.invalidateKBSync(dsid, guid, blob) + Expect(removed).To(BeZero()) + Expect(newStore().cachedKBSync(dsid, guid)).To(Equal(newer)) + }) +}) diff --git a/pkg/appstore/appstore_kbsync_test.go b/pkg/appstore/appstore_kbsync_test.go new file mode 100644 index 00000000..5f43294b --- /dev/null +++ b/pkg/appstore/appstore_kbsync_test.go @@ -0,0 +1,632 @@ +package appstore + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + gohttp "net/http" + "time" + + "github.com/majd/ipatool/v2/pkg/http" + "github.com/majd/ipatool/v2/pkg/keychain" + "github.com/majd/ipatool/v2/pkg/util/machine" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + "howett.net/plist" +) + +const testEntDownloadEndpoint = "https://downloaddispatch.itunes.apple.com/WebObjects/DownloadDispatch.woa/wa/ent/download" + +var _ = Describe("AppStore (preferred kbsync download)", func() { + const guid = "001122334455" + const versionID = "123456789" + var ( + store *appstore + downloads *http.MockClient[downloadResult] + bags *http.MockClient[bagResult] + platforms *http.MockClient[platformVersionLookupResult] + account Account + app App + bag bagResult + empty, unavailable, success http.Result[downloadResult] + generated int + ) + BeforeEach(func() { + ctrl := gomock.NewController(GinkgoT()) + downloads = http.NewMockClient[downloadResult](ctrl) + bags = http.NewMockClient[bagResult](ctrl) + platforms = http.NewMockClient[platformVersionLookupResult](ctrl) + generated = 0 + store = &appstore{ + downloadClient: downloads, bagClient: bags, platformClient: platforms, + kbsyncGenerator: func(ctx context.Context, hardwareID []byte, dsid uint64) ([]byte, error) { + Expect(ctx.Err()).NotTo(HaveOccurred()) + deadline, ok := ctx.Deadline() + Expect(ok).To(BeTrue()) + Expect(time.Until(deadline)).To(BeNumerically("<=", 2*time.Minute)) + Expect(hardwareID).To(Equal([]byte{0, 0x11, 0x22, 0x33, 0x44, 0x55})) + Expect(dsid).To(Equal(uint64(123456789012))) + generated++ + + return []byte("kbsync blob"), nil + }, + } + account = Account{DirectoryServicesID: "123456789012", PasswordToken: "test-token", StoreFront: "143441-1,29"} + app = App{ID: 42, BundleID: "com.example.app"} + bag = bagResult{URLBag: urlBag{EntDownloadEndpoint: testEntDownloadEndpoint}} + empty = http.Result[downloadResult]{StatusCode: gohttp.StatusOK} + unavailable = http.Result[downloadResult]{StatusCode: gohttp.StatusOK, Data: downloadResult{CustomerMessage: "App No Longer Available"}} + success = http.Result[downloadResult]{StatusCode: gohttp.StatusOK, Data: downloadResult{Items: []downloadItemResult{{ + URL: "https://example.com/app.ipa", + Metadata: map[string]interface{}{"itemId": int64(42), "softwareVersionExternalIdentifier": versionID, "softwareVersionBundleId": app.BundleID}, + }}}} + }) + + It("sends kbsync to the full ent endpoint advertised in Apple's bag", func() { + var decoded bagResult + _, err := plist.Unmarshal([]byte(`urlBagvolumeStoreDownloadProduct`+testEntDownloadEndpoint+``), &decoded) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.URLBag.EntDownloadEndpoint).To(Equal(testEntDownloadEndpoint)) + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: decoded}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testEntDownloadEndpoint + "?guid=" + guid)) + Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("kbsync", base64.StdEncoding.EncodeToString([]byte("kbsync blob")))) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(Equal(1)) + }) + + It("enables kbsync generation for normal clients", func() { + Expect(NewAppStore(Args{}).(*appstore).kbsyncGenerator).NotTo(BeNil()) + }) + + It("uses volumeStore when the bag omits ent/download", func() { + bag.URLBag.EntDownloadEndpoint = "" + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(BeZero()) + }) + + It("keeps volumeStore usable when the initial bag request fails", func() { + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{}, errors.New("bag unavailable")), + downloads.EXPECT().Send(gomock.Any()).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(BeZero()) + }) + + It("falls back when generation is unavailable", func() { + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { + return nil, errors.New("guest unavailable") + } + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + }) + + DescribeTable("falls back from unusable ent/download responses", func(outcome string) { + response := empty + var responseErr error + switch outcome { + case "unavailable": + response = unavailable + case "HTML 403": + responseErr = &http.UnexpectedResponseError{StatusCode: gohttp.StatusForbidden} + case "HTTP 500": + responseErr = &http.UnexpectedResponseError{StatusCode: gohttp.StatusInternalServerError} + case "timeout": + responseErr = &http.TransportError{Err: context.DeadlineExceeded} + case "wrong app", "wrong version", "missing URL": + response = http.Result[downloadResult]{StatusCode: gohttp.StatusOK, Data: downloadResult{Items: []downloadItemResult{{ + URL: "https://example.com/wrong.ipa", Metadata: map[string]interface{}{ + "itemId": int64(42), "softwareVersionExternalIdentifier": versionID, "softwareVersionBundleId": app.BundleID, + }, + }}}} + switch outcome { + case "wrong app": + response.Data.Items[0].Metadata["itemId"] = int64(99) + case "wrong version": + response.Data.Items[0].Metadata["softwareVersionExternalIdentifier"] = "other" + default: + response.Data.Items[0].URL = "" + } + } + + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testEntDownloadEndpoint + "?guid=" + guid)) + }).Return(response, responseErr), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("externalVersionId", versionID)) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(Equal(1)) + }, + Entry("empty song list", "empty"), Entry("unavailable", "unavailable"), Entry("HTML 403", "HTML 403"), + Entry("HTTP 500", "HTTP 500"), Entry("timeout", "timeout"), Entry("wrong app", "wrong app"), + Entry("wrong version", "wrong version"), Entry("missing package URL", "missing URL"), + ) + + It("retains the complete legacy chain and reuses the bag after an ent/download miss", func() { + bag.URLBag.RedownloadEndpoint = testRedownloadEndpoint + bag.URLBag.UpdateEndpoint = testUpdateEndpoint + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + downloads.EXPECT().Send(gomock.Any()).Return(unavailable, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + }).Return(empty, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testRedownloadEndpoint + "?guid=" + guid)) + }).Return(unavailable, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testUpdateEndpoint + "?guid=" + guid)) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(Equal(1)) + }) + + DescribeTable("tries ent/download first for a pinned version", + func(platform Platform) { + fetched := bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil) + downloads.EXPECT().Send(gomock.Any()).After(fetched).Do(func(req http.Request) { + Expect(generated).To(Equal(1)) + Expect(req.URL).To(Equal(testEntDownloadEndpoint + "?guid=" + guid)) + Expect(req.Method).To(Equal(http.MethodPOST)) + Expect(req.NoRedirects).To(BeTrue()) + Expect(req.Context.Err()).NotTo(HaveOccurred()) + deadline, ok := req.Context.Deadline() + Expect(ok).To(BeTrue()) + Expect(time.Until(deadline)).To(BeNumerically("<=", 30*time.Second)) + Expect(req.Headers).To(HaveKeyWithValue("X-Token", account.PasswordToken)) + Expect(req.Headers).To(HaveKeyWithValue("X-Dsid", account.DirectoryServicesID)) + Expect(req.Headers).To(HaveKeyWithValue("iCloud-DSID", account.DirectoryServicesID)) + Expect(req.Headers).To(HaveKeyWithValue("X-Apple-Store-Front", account.StoreFront)) + Expect(req.Headers).To(HaveKeyWithValue("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")) + Expect(req.Headers["User-Agent"]).To(HavePrefix("Configurator/2.18 ")) + Expect(req.Payload.(*http.XMLPayload).Content).To(Equal(map[string]interface{}{ + "creditDisplay": "", "guid": guid, "salableAdamId": "42", "externalVersionId": versionID, + "serialNumber": base64.StdEncoding.EncodeToString([]byte{0x54, 0xc8, 0xb0, 0xa9, 0x88, 0x22, 0x33, 0x44, 0x55}), + "kbsync": base64.StdEncoding.EncodeToString([]byte("kbsync blob")), + })) + }).Return(success, nil) + actual, resolved, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, platform) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(resolved).To(Equal(platform)) + }, + Entry("iPhone", PlatformIPhone), Entry("iPad", PlatformIPad), Entry("macOS", PlatformMacOS), + Entry("tvOS", PlatformAppleTV), Entry("visionOS", PlatformVisionOS), + ) + + It("pins the iOS offer before trying ent/download", func() { + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + platforms.EXPECT().Send(gomock.Any()).Return(http.Result[platformVersionLookupResult]{StatusCode: gohttp.StatusOK, Data: platformVersionLookupResult{ + Results: map[string]platformVersionLookupItem{"42": {Offers: []platformVersionLookupOffer{{Version: platformVersionLookupVersion{ExternalID: platformVersionExternalID(versionID)}}}}}, + }}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("externalVersionId", versionID)) + }).Return(success, nil), + ) + actual, platform, err := store.sendDownloadProduct(context.Background(), account, app, guid, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(platform).To(Equal(PlatformIPhone)) + }) + + It("pins the native Mac offer before trying ent/download for a universal app", func() { + pages := http.NewMockClient[[]byte](gomock.NewController(GinkgoT())) + store.storefrontClient = pages + account.StoreFront = "143443-2,34" + app = App{ID: 6472431552, BundleID: "com.nebula.karing"} + success.Data.Items[0].Metadata = map[string]interface{}{ + "itemId": app.ID, "softwareVersionExternalIdentifier": "876660716", "softwareVersionBundleId": app.BundleID, + } + gomock.InOrder( + pages.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{StatusCode: gohttp.StatusOK, Data: macVersionPage(karingMacConfiguration)}, nil), + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testEntDownloadEndpoint + "?guid=" + guid)) + Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("externalVersionId", "876660716")) + }).Return(success, nil), + ) + actual, platform, err := store.sendDownloadProduct(context.Background(), account, app, guid, "", PlatformMacOS) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(platform).To(Equal(PlatformMacOS)) + }) + + It("uses ent/download when listing versions", func() { + device := machine.NewMockMachine(gomock.NewController(GinkgoT())) + store.machine = device + device.EXPECT().MacAddress().Return("00:11:22:33:44:55", nil) + success.Data.Items[0].Metadata["softwareVersionExternalIdentifiers"] = []interface{}{"123456788", versionID} + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + platforms.EXPECT().Send(gomock.Any()).Return(http.Result[platformVersionLookupResult]{StatusCode: gohttp.StatusOK, Data: platformVersionLookupResult{ + Results: map[string]platformVersionLookupItem{"42": {Offers: []platformVersionLookupOffer{{Version: platformVersionLookupVersion{ExternalID: platformVersionExternalID(versionID)}}}}}, + }}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testEntDownloadEndpoint + "?guid=" + guid)) + }).Return(success, nil), + ) + result, err := store.ListVersions(ListVersionsInput{Account: account, App: app}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.ExternalVersionIdentifiers).To(Equal([]string{"123456788", versionID})) + Expect(result.LatestExternalVersionID).To(Equal(versionID)) + }) + + DescribeTable("rejects invalid bag destinations before generating or sending credentials", func(endpoint string) { + bag.URLBag.EntDownloadEndpoint = endpoint + _, _, err := store.sendPreferredDownload(context.Background(), bag.URLBag.EntDownloadEndpoint, account, app, guid, versionID, PlatformIPhone) + Expect(err).To(MatchError("invalid download endpoint in bag")) + Expect(generated).To(BeZero()) + }, + Entry("HTTP", "http://downloaddispatch.itunes.apple.com/WebObjects/DownloadDispatch.woa/wa/ent/download"), + Entry("foreign host", "https://example.com/WebObjects/DownloadDispatch.woa/wa/ent/download"), + Entry("lookalike host", "https://downloaddispatch.itunes.apple.com.example.com/WebObjects/DownloadDispatch.woa/wa/ent/download"), + Entry("credentials", "https://user@downloaddispatch.itunes.apple.com/WebObjects/DownloadDispatch.woa/wa/ent/download"), + Entry("query", testEntDownloadEndpoint+"?guid=other"), Entry("fragment", testEntDownloadEndpoint+"#fragment"), + Entry("wrong path", testUpdateEndpoint), + Entry("abbreviated path", "https://downloaddispatch.itunes.apple.com/ent/download"), + Entry("encoded path", "https://downloaddispatch.itunes.apple.com/WebObjects/DownloadDispatch.woa/wa/ent/%64ownload"), + ) + + DescribeTable("validates the app, version and bundle before accepting ent/download", func(field string, value interface{}, expected string) { + success.Data.Items[0].Metadata[field] = value + downloads.EXPECT().Send(gomock.Any()).Return(success, nil) + _, _, err := store.sendPreferredDownload(context.Background(), bag.URLBag.EntDownloadEndpoint, account, app, guid, versionID, PlatformIPhone) + Expect(err).To(MatchError(ContainSubstring(expected))) + }, + Entry("wrong app", "itemId", int64(99), "requested app or version"), + Entry("wrong version", "softwareVersionExternalIdentifier", "987654321", "requested app or version"), + Entry("wrong bundle", "softwareVersionBundleId", "com.other.app", "requested bundle identifier"), + Entry("missing bundle", "softwareVersionBundleId", nil, "requested bundle identifier"), + ) + + DescribeTable("preserves authentication and license failures from the fallback", func(failure string) { + response := http.Result[downloadResult]{StatusCode: gohttp.StatusOK, Data: downloadResult{FailureType: failure}} + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil) + gomock.InOrder( + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testEntDownloadEndpoint + "?guid=" + guid)) + }).Return(response, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + }).Return(response, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(response)) + }, Entry("expired token", FailureTypePasswordTokenExpired), Entry("missing license", FailureTypeLicenseNotFound)) + + It("stops before generating when canceled", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _, err := store.sendDownloadProduct(ctx, account, app, guid, versionID, PlatformIPhone) + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + Expect(generated).To(BeZero()) + }) + + It("reports generation failures without sending the token", func() { + failure := errors.New("guest failed") + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { return nil, failure } + _, _, err := store.sendPreferredDownload(context.Background(), bag.URLBag.EntDownloadEndpoint, account, app, guid, versionID, PlatformIPhone) + Expect(errors.Is(err, failure)).To(BeTrue()) + }) + + It("does not start the legacy chain after cancellation during generation", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil) + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { + cancel() + + return nil, context.Canceled + } + _, _, err := store.sendDownloadProduct(ctx, account, app, guid, versionID, PlatformIPhone) + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + }) + + It("preserves unpinned legacy downloads when the preferred offer lookup fails", func() { + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + platforms.EXPECT().Send(gomock.Any()).Return(http.Result[platformVersionLookupResult]{}, errors.New("catalog unavailable")), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + Expect(req.Payload.(*http.XMLPayload).Content).NotTo(HaveKey("externalVersionId")) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, "", PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(BeZero()) + }) + + It("does not send credentials to an invalid preferred endpoint and falls back", func() { + bag.URLBag.EntDownloadEndpoint = "https://example.com/WebObjects/DownloadDispatch.woa/wa/ent/download" + gomock.InOrder( + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(BeZero()) + }) + + Describe("retrying cached kbsync", func() { + const dsid = uint64(123456789012) + var ( + stored []byte + cachedBlob, freshBlob string + removed, saved int + removeErr error + request func(string) *gomock.Call + failure func(string) (http.Result[downloadResult], error) + ) + BeforeEach(func() { + cachedBlob = base64.StdEncoding.EncodeToString([]byte("cached blob")) + freshBlob = base64.StdEncoding.EncodeToString([]byte("kbsync blob")) + var err error + stored, err = json.Marshal(kbsyncCacheEntry{DSID: dsid, GUID: guid, Data: cachedBlob}) + Expect(err).NotTo(HaveOccurred()) + removed, saved = 0, 0 + removeErr = nil + chain := keychain.NewMockKeychain(gomock.NewController(GinkgoT())) + store.keychain = chain + chain.EXPECT().Get(kbsyncCacheKey).DoAndReturn(func(string) ([]byte, error) { + return bytes.Clone(stored), nil + }).AnyTimes() + chain.EXPECT().Remove(kbsyncCacheKey).DoAndReturn(func(string) error { + removed++ + if removeErr != nil { + return removeErr + } + stored = nil + + return nil + }).AnyTimes() + chain.EXPECT().Set(kbsyncCacheKey, gomock.Any()).DoAndReturn(func(_ string, data []byte) error { + saved++ + stored = bytes.Clone(data) + + return nil + }).AnyTimes() + bags.EXPECT().Send(gomock.Any()).Return(http.Result[bagResult]{StatusCode: gohttp.StatusOK, Data: bag}, nil).AnyTimes() + request = func(blob string) *gomock.Call { + return downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(Equal(testEntDownloadEndpoint + "?guid=" + guid)) + Expect(req.NoRedirects).To(BeTrue()) + Expect(req.Context.Err()).NotTo(HaveOccurred()) + Expect(req.Headers).To(HaveKeyWithValue("X-Token", account.PasswordToken)) + payload := req.Payload.(*http.XMLPayload).Content + Expect(payload).To(HaveKeyWithValue("kbsync", blob)) + Expect(payload).To(HaveKeyWithValue("externalVersionId", versionID)) + }) + } + failure = func(outcome string) (http.Result[downloadResult], error) { + switch outcome { + case "HTTP 500": + return http.Result[downloadResult]{StatusCode: gohttp.StatusInternalServerError}, nil + case "HTML 403": + return http.Result[downloadResult]{}, &http.UnexpectedResponseError{StatusCode: gohttp.StatusForbidden} + case "timeout": + return http.Result[downloadResult]{}, &http.TransportError{Err: context.DeadlineExceeded} + case "unavailable": + return unavailable, nil + case "structured failure": + return http.Result[downloadResult]{StatusCode: gohttp.StatusOK, Data: downloadResult{FailureType: FailureTypePasswordTokenExpired}}, nil + case "missing URL", "wrong version": + response := http.Result[downloadResult]{StatusCode: gohttp.StatusOK, Data: downloadResult{Items: []downloadItemResult{{ + Metadata: map[string]interface{}{"itemId": app.ID, "softwareVersionExternalIdentifier": versionID, "softwareVersionBundleId": app.BundleID}, + }}}} + if outcome == "wrong version" { + response.Data.Items[0].URL = "https://example.com/wrong.ipa" + response.Data.Items[0].Metadata["softwareVersionExternalIdentifier"] = "other" + } + + return response, nil + default: + return empty, nil + } + } + }) + + It("uses a working persistent blob without generating or rewriting it", func() { + request(cachedBlob).Return(success, nil) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(BeZero()) + Expect(removed).To(BeZero()) + Expect(saved).To(BeZero()) + }) + + It("saves an uncached blob only after success and reuses it in a new client", func() { + stored = nil + request(freshBlob).Do(func(http.Request) { + Expect(stored).To(BeNil()) + Expect(saved).To(BeZero()) + }).Return(success, nil) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(saved).To(Equal(1)) + restarted := &appstore{ + keychain: store.keychain, downloadClient: downloads, bagClient: bags, + kbsyncGenerator: func(context.Context, []byte, uint64) ([]byte, error) { + Fail("the new client must reuse the successful blob") + + return nil, nil + }, + } + request(freshBlob).Return(success, nil) + _, _, err = restarted.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(generated).To(Equal(1)) + Expect(saved).To(Equal(1)) + }) + + DescribeTable("retries a failed cached blob and saves a successful replacement", func(outcome string) { + response, responseErr := failure(outcome) + var firstContext context.Context + gomock.InOrder( + request(cachedBlob).Do(func(req http.Request) { firstContext = req.Context }).Return(response, responseErr), + request(freshBlob).Do(func(req http.Request) { + Expect(removed).To(Equal(1)) + Expect(stored).To(BeNil()) + Expect(saved).To(BeZero()) + Expect(firstContext.Err()).To(HaveOccurred()) + Expect(req.Context).NotTo(Equal(firstContext)) + Expect(store.cachedKBSync(dsid, guid)).To(BeEmpty()) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(Equal(1)) + Expect(saved).To(Equal(1)) + var entry kbsyncCacheEntry + Expect(json.Unmarshal(stored, &entry)).To(Succeed()) + Expect(entry).To(Equal(kbsyncCacheEntry{DSID: dsid, GUID: guid, Data: freshBlob})) + Expect(store.cachedKBSync(dsid, guid)).To(Equal(freshBlob)) + request(freshBlob).Return(success, nil) + _, _, err = store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(generated).To(Equal(1)) + Expect(saved).To(Equal(1)) + }, + Entry("server error", "HTTP 500"), Entry("non-plist error", "HTML 403"), Entry("request timeout", "timeout"), + Entry("empty response", "empty"), Entry("availability error", "unavailable"), Entry("Apple failure", "structured failure"), + Entry("missing URL", "missing URL"), Entry("wrong version", "wrong version"), + ) + + DescribeTable("falls back without caching or retrying a failed fresh blob", func(outcome string, hasCache bool) { + if hasCache { + request(cachedBlob).Return(empty, nil) + } else { + stored = nil + } + response, responseErr := failure(outcome) + gomock.InOrder( + request(freshBlob).Return(response, responseErr), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(Equal(1)) + Expect(saved).To(BeZero()) + Expect(stored).To(BeNil()) + Expect(store.cachedKBSync(dsid, guid)).To(BeEmpty()) + }, + Entry("server error", "HTTP 500", true), Entry("non-plist error", "HTML 403", true), Entry("timeout", "timeout", true), + Entry("empty response", "empty", true), Entry("availability error", "unavailable", true), + Entry("Apple failure", "structured failure", true), Entry("missing URL", "missing URL", true), Entry("wrong version", "wrong version", true), + Entry("no existing cache", "HTTP 500", false), Entry("uncached Apple failure", "structured failure", false), + ) + + It("falls back when regeneration fails", func() { + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { + Expect(removed).To(Equal(1)) + + return nil, errors.New("guest failed") + } + gomock.InOrder( + request(cachedBlob).Return(empty, nil), + downloads.EXPECT().Send(gomock.Any()).Do(func(req http.Request) { + Expect(req.URL).To(ContainSubstring("volumeStoreDownloadProduct")) + }).Return(success, nil), + ) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(saved).To(BeZero()) + Expect(stored).To(BeNil()) + }) + + It("bypasses the old blob even if its keychain entry cannot be removed", func() { + removeErr = errors.New("keychain unavailable") + gomock.InOrder(request(cachedBlob).Return(empty, nil), request(freshBlob).Return(success, nil)) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, guid, versionID, PlatformIPhone) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(success)) + Expect(generated).To(Equal(1)) + Expect(removed).To(Equal(1)) + Expect(saved).To(Equal(1)) + Expect(store.cachedKBSync(dsid, guid)).To(Equal(freshBlob)) + }) + + It("does not regenerate or fall back when the cached request is canceled", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request(cachedBlob).Do(func(http.Request) { cancel() }).Return(http.Result[downloadResult]{}, context.Canceled) + _, _, err := store.sendDownloadProduct(ctx, account, app, guid, versionID, PlatformIPhone) + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + Expect(generated).To(BeZero()) + Expect(removed).To(BeZero()) + Expect(saved).To(BeZero()) + }) + + It("does not use the cached blob when already canceled", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _, err := store.sendPreferredDownload(ctx, testEntDownloadEndpoint, account, app, guid, versionID, PlatformIPhone) + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + Expect(generated).To(BeZero()) + Expect(removed).To(BeZero()) + }) + + It("stops without saving or falling back when regeneration is canceled", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request(cachedBlob).Return(empty, nil) + store.kbsyncGenerator = func(context.Context, []byte, uint64) ([]byte, error) { + cancel() + + return []byte("canceled blob"), nil + } + _, _, err := store.sendDownloadProduct(ctx, account, app, guid, versionID, PlatformIPhone) + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + Expect(removed).To(Equal(1)) + Expect(saved).To(BeZero()) + Expect(stored).To(BeNil()) + }) + }) +}) diff --git a/pkg/appstore/appstore_list_versions.go b/pkg/appstore/appstore_list_versions.go index f682989b..bedf9461 100644 --- a/pkg/appstore/appstore_list_versions.go +++ b/pkg/appstore/appstore_list_versions.go @@ -1,12 +1,14 @@ package appstore import ( + "context" "errors" "fmt" "strings" ) type ListVersionsInput struct { + Context context.Context Account Account App App Platform Platform @@ -49,7 +51,7 @@ func (t *appstore) ListVersions(input ListVersionsInput) (ListVersionsOutput, er return ListVersionsOutput{}, fmt.Errorf("failed to resolve platform version: %w", err) } - res, _, err := t.sendDownloadProduct(input.Account, input.App, guid, externalVersionID, platform) + res, _, err := t.sendDownloadProduct(input.Context, input.Account, input.App, guid, externalVersionID, platform) if err != nil { return ListVersionsOutput{}, err } diff --git a/pkg/appstore/appstore_login.go b/pkg/appstore/appstore_login.go index aa2636e8..6ac88727 100644 --- a/pkg/appstore/appstore_login.go +++ b/pkg/appstore/appstore_login.go @@ -318,7 +318,10 @@ func retryableAuthenticationError(err error) (int, bool) { } status := responseErr.StatusCode + // A Store pod can transiently return an HTML 403 page. Populated Apple + // credential errors are decoded normally and never reach this branch. retry := status == gohttp.StatusNoContent || + status == gohttp.StatusForbidden || status == gohttp.StatusNotFound || status == gohttp.StatusTooManyRequests || status/100 == 5 diff --git a/pkg/appstore/appstore_login_test.go b/pkg/appstore/appstore_login_test.go index a41227ff..ac2281fa 100644 --- a/pkg/appstore/appstore_login_test.go +++ b/pkg/appstore/appstore_login_test.go @@ -125,7 +125,8 @@ var _ = Describe("AppStore (Login)", func() { }, Entry("retries and then stops for HTTP 204", 204, maxAuthenticationRequestAttempts), Entry("retries and then stops for HTTP 503", 503, maxAuthenticationRequestAttempts), - Entry("does not retry HTTP 403", 403, 1), + Entry("retries and then stops for non-plist HTTP 403", 403, maxAuthenticationRequestAttempts), + Entry("does not retry HTTP 401", 401, 1), ) }) diff --git a/pkg/appstore/appstore_macos_version_lookup_test.go b/pkg/appstore/appstore_macos_version_lookup_test.go index d2076518..9cfff5a7 100644 --- a/pkg/appstore/appstore_macos_version_lookup_test.go +++ b/pkg/appstore/appstore_macos_version_lookup_test.go @@ -1,6 +1,7 @@ package appstore import ( + "context" "errors" gohttp "net/http" @@ -54,7 +55,7 @@ var _ = Describe("Mac purchase version selection", func() { Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("externalVersionId", "876660716")) }).Return(expected, downloadErr), ) - output, platform, err := store.sendDownloadProduct(Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) + output, platform, err := store.sendDownloadProduct(context.Background(), Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) Expect(output).To(Equal(expected)) Expect(platform).To(Equal(PlatformMacOS)) if downloadErr == nil { @@ -88,7 +89,7 @@ var _ = Describe("Mac purchase version selection", func() { Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("appExtVrsId", "876660716")) }).Return(expected, nil), ) - output, platform, err := store.sendDownloadProduct(Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) + output, platform, err := store.sendDownloadProduct(context.Background(), Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) Expect(err).ToNot(HaveOccurred()) Expect(output).To(Equal(expected)) Expect(platform).To(Equal(PlatformMacOS)) @@ -103,7 +104,7 @@ var _ = Describe("Mac purchase version selection", func() { pages := http.NewMockClient[[]byte](ctrl) store := &appstore{storefrontClient: pages, downloadClient: http.NewMockClient[downloadResult](ctrl)} pages.EXPECT().Send(gomock.Any()).Return(http.Result[[]byte]{StatusCode: status, Data: body}, requestErr) - _, _, err := store.sendDownloadProduct(Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) + _, _, err := store.sendDownloadProduct(context.Background(), Account{StoreFront: "143443-2,34"}, app, "001122334455", "", PlatformMacOS) Expect(err).To(HaveOccurred()) }, Entry("HTTP failure", 500, nil, nil), diff --git a/pkg/appstore/appstore_revoke.go b/pkg/appstore/appstore_revoke.go index 154d88da..ad1d9e32 100644 --- a/pkg/appstore/appstore_revoke.go +++ b/pkg/appstore/appstore_revoke.go @@ -1,14 +1,30 @@ package appstore import ( + "errors" "fmt" + "io/fs" + + "github.com/byteness/keyring" ) func (t *appstore) Revoke() error { - err := t.keychain.Remove("account") - if err != nil { - return fmt.Errorf("failed to remove account from keychain: %w", err) + accountErr := t.keychain.Remove("account") + if accountErr != nil { + accountErr = fmt.Errorf("failed to remove account from keychain: %w", accountErr) + } + + t.kbsyncCache.mu.Lock() + defer t.kbsyncCache.mu.Unlock() + + t.kbsyncCache.entry = kbsyncCacheEntry{} + + cacheErr := t.keychain.Remove(kbsyncCacheKey) + if errors.Is(cacheErr, keyring.ErrKeyNotFound) || errors.Is(cacheErr, fs.ErrNotExist) { + cacheErr = nil + } else if cacheErr != nil { + cacheErr = fmt.Errorf("failed to remove kbsync cache from keychain: %w", cacheErr) } - return nil + return errors.Join(accountErr, cacheErr) } diff --git a/pkg/appstore/appstore_revoke_test.go b/pkg/appstore/appstore_revoke_test.go index 4c4257d7..82a6458f 100644 --- a/pkg/appstore/appstore_revoke_test.go +++ b/pkg/appstore/appstore_revoke_test.go @@ -2,7 +2,9 @@ package appstore import ( "errors" + "io/fs" + "github.com/byteness/keyring" "github.com/majd/ipatool/v2/pkg/keychain" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,6 +35,7 @@ var _ = Describe("AppStore (Revoke)", func() { mockKeychain.EXPECT(). Remove("account"). Return(nil) + mockKeychain.EXPECT().Remove(kbsyncCacheKey).Return(nil) }) It("returns data", func() { @@ -41,11 +44,27 @@ var _ = Describe("AppStore (Revoke)", func() { }) }) + DescribeTable("revokes accounts saved before a kbsync cache existed", func(missing error) { + mockKeychain.EXPECT().Remove("account").Return(nil) + mockKeychain.EXPECT().Remove(kbsyncCacheKey).Return(missing) + Expect(appstore.Revoke()).To(Succeed()) + }, + Entry("native keychain", keyring.ErrKeyNotFound), + Entry("file keychain", &fs.PathError{Op: "remove", Path: "cache", Err: fs.ErrNotExist}), + ) + + It("reports a cache removal failure after removing the account", func() { + mockKeychain.EXPECT().Remove("account").Return(nil) + mockKeychain.EXPECT().Remove(kbsyncCacheKey).Return(errors.New("cache unavailable")) + Expect(appstore.Revoke()).To(MatchError("failed to remove kbsync cache from keychain: cache unavailable")) + }) + When("keychain returns error", func() { BeforeEach(func() { mockKeychain.EXPECT(). Remove("account"). Return(errors.New("")) + mockKeychain.EXPECT().Remove(kbsyncCacheKey).Return(nil) }) It("returns wrapped error", func() { diff --git a/pkg/appstore/appstore_update_product_test.go b/pkg/appstore/appstore_update_product_test.go index c4bdc8d8..4ea890b9 100644 --- a/pkg/appstore/appstore_update_product_test.go +++ b/pkg/appstore/appstore_update_product_test.go @@ -1,6 +1,7 @@ package appstore import ( + "context" "errors" "fmt" gohttp "net/http" @@ -102,7 +103,7 @@ var _ = Describe("AppStore (Update Product)", func() { }).Return(expected, nil), ) - actual, resolvedPlatform, err := store.sendDownloadProduct(account, app, testGUID, versionID, platform) + actual, resolvedPlatform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, versionID, platform) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(expected)) if platform == "" { @@ -150,7 +151,7 @@ var _ = Describe("AppStore (Update Product)", func() { }).Return(update, nil), ) - actual, resolvedPlatform, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, platform) + actual, resolvedPlatform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, platform) Expect(resolvedPlatform).To(Equal(platform)) Expect(actual).To(Equal(update)) if updateUnavailable { @@ -212,7 +213,7 @@ var _ = Describe("AppStore (Update Product)", func() { Expect(req.Payload.(*http.XMLPayload).Content).To(HaveKeyWithValue("appExtVrsId", testVersionID)) }).Return(update, nil), ) - actual, platform, err := store.sendDownloadProduct(account, app, testGUID, versionID, PlatformMacOS) + actual, platform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, versionID, PlatformMacOS) Expect(platform).To(Equal(PlatformMacOS)) Expect(actual).To(Equal(update)) if updateUnavailable { @@ -235,7 +236,7 @@ var _ = Describe("AppStore (Update Product)", func() { Data: downloadResult{FailureType: failureType, CustomerMessage: message}} mockDownloadClient.EXPECT().Send(gomock.Any()).After(previous).Return(response, nil) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, platform) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, platform) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(response)) }, @@ -252,7 +253,7 @@ var _ = Describe("AppStore (Update Product)", func() { mockDownloadClient.EXPECT().Send(gomock.Any()).After(previous). Return(http.Result[downloadResult]{}, original) - _, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(errors.Is(err, original)).To(BeTrue()) }, Entry("authentication", &http.UnexpectedResponseError{StatusCode: gohttp.StatusForbidden}), @@ -267,7 +268,7 @@ var _ = Describe("AppStore (Update Product)", func() { previous := expectPrimary() mockDownloadClient.EXPECT().Send(gomock.Any()).After(previous).Return(response, nil) - actual, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + actual, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(response)) }, @@ -285,7 +286,7 @@ var _ = Describe("AppStore (Update Product)", func() { mockDownloadClient.EXPECT().Send(gomock.Any()).After(previous). Return(http.Result[downloadResult]{}, redownloadErr) - _, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(errors.Is(err, redownloadErr)).To(BeTrue()) }) @@ -295,7 +296,7 @@ var _ = Describe("AppStore (Update Product)", func() { mockDownloadClient.EXPECT().Send(gomock.Any()).After(previous). Return(http.Result[downloadResult]{}, redownloadErr) - _, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(err).To(MatchError("invalid download endpoint in bag")) }) @@ -305,7 +306,7 @@ var _ = Describe("AppStore (Update Product)", func() { mockDownloadClient.EXPECT().Send(gomock.Any()).After(previous). Return(http.Result[downloadResult]{}, redownloadErr) - _, resolvedPlatform, err := store.sendDownloadProduct(account, app, testGUID, versionID, platform) + _, resolvedPlatform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, versionID, platform) Expect(errors.Is(err, redownloadErr)).To(BeTrue()) Expect(resolvedPlatform).To(Equal(platform)) }, @@ -319,7 +320,7 @@ var _ = Describe("AppStore (Update Product)", func() { Data: downloadResult{CustomerMessage: "No Longer Available"}} mockDownloadClient.EXPECT().Send(gomock.Any()).After(previous).Return(response, nil) - actual, platform, err := store.sendDownloadProduct(account, app, testGUID, "", PlatformAppleTV) + actual, platform, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, "", PlatformAppleTV) Expect(err).ToNot(HaveOccurred()) Expect(actual).To(Equal(response)) Expect(platform).To(Equal(PlatformAppleTV)) @@ -334,7 +335,7 @@ var _ = Describe("AppStore (Update Product)", func() { Return(http.Result[downloadResult]{}, redownloadErr), ) - _, _, err := store.sendDownloadProduct(account, app, testGUID, testVersionID, PlatformIPhone) + _, _, err := store.sendDownloadProduct(context.Background(), account, app, testGUID, testVersionID, PlatformIPhone) Expect(errors.Is(err, redownloadErr)).To(BeTrue()) Expect(err).To(MatchError(ContainSubstring("failed to send update request"))) }) diff --git a/pkg/appstore/authentication_retry_test.go b/pkg/appstore/authentication_retry_test.go index 35e03249..73d817b7 100644 --- a/pkg/appstore/authentication_retry_test.go +++ b/pkg/appstore/authentication_retry_test.go @@ -63,13 +63,13 @@ var _ = Describe("Authentication recovery", func() { Entry("other transport failure", errors.New("unrecoverable transport error"), true, 1), ) - It("shares the attempt budget with HTTP failures and still honors Retry-After", func() { + DescribeTable("shares the attempt budget with HTTP failures and still honors Retry-After", func(status int) { ctrl := gomock.NewController(GinkgoT()) client := apphttp.NewMockClient[loginResult](ctrl) last := &apphttp.TransportError{Err: context.DeadlineExceeded} gomock.InOrder( client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{}, &apphttp.TransportError{Err: io.EOF}), - client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{}, &apphttp.UnexpectedResponseError{StatusCode: 503, RetryAfter: "2"}), + client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{}, &apphttp.UnexpectedResponseError{StatusCode: status, RetryAfter: "2"}), client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{}, last), ) var waits []time.Duration @@ -77,8 +77,11 @@ var _ = Describe("Authentication recovery", func() { _, err := sut.sendAuthenticationRequest(apphttp.Request{}) Expect(errors.Is(err, last)).To(BeTrue()) Expect(waits).To(Equal([]time.Duration{10 * time.Second, 2 * time.Second})) - Expect(err.Error()).To(ContainSubstring("after 3 attempts (transport error, HTTP 503, transport error)")) - }) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("after 3 attempts (transport error, HTTP %d, transport error)", status))) + }, + Entry("non-plist 403", http.StatusForbidden), + Entry("503", http.StatusServiceUnavailable), + ) It("stops when an incomplete response requests a wait beyond the budget", func() { ctrl := gomock.NewController(GinkgoT()) @@ -116,7 +119,9 @@ var _ = Describe("Authentication recovery", func() { Entry("404 without a deadline", 404, "", 3, []time.Duration{10 * time.Second, 20 * time.Second}), Entry("Retry-After zero avoids a tight retry loop", 429, "0", 3, []time.Duration{time.Second, time.Second}), Entry("503 with a long deadline", 503, "3600", 1, []time.Duration(nil)), - Entry("403 is not retried", 403, "1", 1, []time.Duration(nil)), + Entry("403 without a deadline", 403, "", 3, []time.Duration{10 * time.Second, 20 * time.Second}), + Entry("403 with a requested delay", 403, "1", 3, []time.Duration{time.Second, time.Second}), + Entry("403 with a long deadline", 403, "3600", 1, []time.Duration(nil)), Entry("malformed redirect is not retried", 302, "", 1, []time.Duration(nil)), ) @@ -138,7 +143,7 @@ var _ = Describe("Authentication recovery", func() { } }) - It("does not retry populated Apple credential errors or 2FA challenges", func() { + DescribeTable("does not retry populated Apple credential errors or 2FA challenges", func(status int) { ctrl := gomock.NewController(GinkgoT()) client := apphttp.NewMockClient[loginResult](ctrl) sut := &appstore{loginClient: client, authRetrySleep: func(time.Duration) { Fail("must not sleep") }} @@ -146,14 +151,17 @@ var _ = Describe("Authentication recovery", func() { {FailureType: FailureTypeInvalidCredentials, CustomerMessage: "invalid credentials"}, {CustomerMessage: CustomerMessageBadLogin}, } { - client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{StatusCode: 200, Data: data}, nil).Times(1) + client.EXPECT().Send(gomock.Any()).Return(apphttp.Result[loginResult]{StatusCode: status, Data: data}, nil).Times(1) result, err := sut.sendAuthenticationRequest(apphttp.Request{}) Expect(err).NotTo(HaveOccurred()) _, _, err = sut.parseLoginResponse(&result, 2, "", testAuthEndpoint) Expect(err).To(HaveOccurred()) Expect(err.Error()).NotTo(ContainSubstring("another network")) } - }) + }, + Entry("HTTP 200", http.StatusOK), + Entry("HTTP 403", http.StatusForbidden), + ) }) // The adapter only redirects network traffic to the local server; login still @@ -212,7 +220,7 @@ var _ = Describe("Login session continuity", func() { Expect(<-bodies).To(Equal(<-bodies)) }) - It("retains cookies and the signed payload through retries, a pod redirect and 2FA", func() { + It("retains cookies and the signed payload through retries, a pod HTML 403 and 2FA", func() { ctrl := gomock.NewController(GinkgoT()) jar, err := cookiejar.New(nil) Expect(err).NotTo(HaveOccurred()) @@ -220,9 +228,11 @@ var _ = Describe("Login session continuity", func() { adapter := apphttp.NewMockClient[loginResult](ctrl) keychain := keychain.NewMockKeychain(ctrl) keychain.EXPECT().Set("account", gomock.Any()).Return(nil).Times(1) - sut := &appstore{loginClient: adapter, keychain: keychain, authRetrySleep: func(time.Duration) {}} - connections := make(chan string, 4) + var waits []time.Duration + sut := &appstore{loginClient: adapter, keychain: keychain, authRetrySleep: func(delay time.Duration) { waits = append(waits, delay) }} + connections := make(chan string, 5) calls := 0 + var originalBody []byte podURL := "https://p7-buy.itunes.apple.com" + PrivateAppStoreAPIPathAuth srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer GinkgoRecover() @@ -231,6 +241,11 @@ var _ = Describe("Login session continuity", func() { Expect(r.Method).To(Equal(http.MethodPost)) body, err := io.ReadAll(r.Body) Expect(err).NotTo(HaveOccurred()) + if calls == 1 { + originalBody = body + } else if calls <= 4 { + Expect(body).To(Equal(originalBody)) + } signature, err := base64.StdEncoding.DecodeString(r.Header.Get(apphttp.HeaderAppleActionSignature)) Expect(err).NotTo(HaveOccurred()) Expect(signature).To(Equal(body)) @@ -251,12 +266,17 @@ var _ = Describe("Login session continuity", func() { w.Header().Set("Location", podURL) w.WriteHeader(http.StatusFound) case 3: + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusForbidden) + _, err = w.Write([]byte("Access Denied")) + Expect(err).NotTo(HaveOccurred()) + case 4: Expect(payload["password"]).To(Equal("password")) data, err := plist.Marshal(map[string]string{"customerMessage": CustomerMessageBadLogin}, plist.XMLFormat) Expect(err).NotTo(HaveOccurred()) _, err = w.Write(data) Expect(err).NotTo(HaveOccurred()) - case 4: + case 5: Expect(payload["password"]).To(Equal("password123456")) w.Header().Set(HTTPHeaderStoreFront, "143441-1,29") _, err = w.Write([]byte("passwordTokentokendsPersonId123")) @@ -269,23 +289,25 @@ var _ = Describe("Login session continuity", func() { adapted := 0 adapter.EXPECT().Send(gomock.Any()).DoAndReturn(func(request apphttp.Request) (apphttp.Result[loginResult], error) { adapted++ - if adapted == 3 { + if adapted == 3 || adapted == 4 { Expect(request.URL).To(Equal(podURL)) } request.URL = srv.URL + PrivateAppStoreAPIPathAuth return client.Send(request) - }).Times(4) + }).Times(5) signer := &stubActionSigner{} _, err = sut.login("email", "password", "", "guid", testAuthEndpoint, signer) Expect(errors.Is(err, ErrAuthCodeRequired)).To(BeTrue()) account, err := sut.login("email", "password", "123456", "guid", testAuthEndpoint, signer) Expect(err).NotTo(HaveOccurred()) Expect(account.PasswordToken).To(Equal("token")) + Expect(waits).To(Equal([]time.Duration{10 * time.Second, 10 * time.Second})) + Expect(signer.signCalls).To(Equal(5)) seen := map[string]bool{} - for range 4 { + for range 5 { seen[<-connections] = true } - Expect(seen).To(HaveLen(4)) + Expect(seen).To(HaveLen(5)) }) }) diff --git a/pkg/http/client.go b/pkg/http/client.go index 31d63180..5b0da730 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -157,6 +157,10 @@ func (c *client[R]) Send(req Request) (Result[R], error) { return Result[R]{}, fmt.Errorf("failed to create request: %w", err) } + if req.Context != nil { + request = request.WithContext(req.Context) + } + for key, val := range req.Headers { request.Header.Set(key, val) } @@ -170,7 +174,12 @@ func (c *client[R]) Send(req Request) (Result[R], error) { request.Header.Set(HeaderAppleActionSignature, base64.StdEncoding.EncodeToString(signature)) } - res, err := c.internalClient.Do(request) + client := c.internalClient + if req.NoRedirects { + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + } + + res, err := client.Do(request) if err != nil { return Result[R]{}, fmt.Errorf("request failed: %w", &TransportError{Err: err}) } diff --git a/pkg/http/request.go b/pkg/http/request.go index a8d8ba47..1d8c5e42 100644 --- a/pkg/http/request.go +++ b/pkg/http/request.go @@ -1,6 +1,10 @@ package http +import "context" + type Request struct { + Context context.Context + NoRedirects bool Method string URL string Headers map[string]string diff --git a/pkg/http/request_options_test.go b/pkg/http/request_options_test.go new file mode 100644 index 00000000..3a521b0f --- /dev/null +++ b/pkg/http/request_options_test.go @@ -0,0 +1,75 @@ +package http + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" +) + +var _ = Describe("HTTP request options", func() { + var jar *MockCookieJar + BeforeEach(func() { + jar = NewMockCookieJar(gomock.NewController(GinkgoT())) + jar.EXPECT().Cookies(gomock.Any()).Return(nil).AnyTimes() + jar.EXPECT().Save().Return(nil).AnyTimes() + }) + + It("stops redirects for a token-bearing request without changing subsequent requests", func() { + var visited atomic.Int32 + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + visited.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer destination.Close() + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, destination.URL, http.StatusTemporaryRedirect) + })) + defer source.Close() + client := NewClient[[]byte](Args{CookieJar: jar}) + request := Request{URL: source.URL, Method: MethodPOST, ResponseFormat: ResponseFormatRaw, NoRedirects: true, Headers: map[string]string{"X-Token": "test-token"}} + result, err := client.Send(request) + Expect(err).NotTo(HaveOccurred()) + Expect(result.StatusCode).To(Equal(http.StatusTemporaryRedirect)) + Expect(visited.Load()).To(BeZero()) + + result, err = client.Send(Request{URL: source.URL, Method: MethodGET, ResponseFormat: ResponseFormatRaw}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.StatusCode).To(Equal(http.StatusOK)) + Expect(visited.Load()).To(Equal(int32(1))) + }) + + It("honors a request deadline while reading the response body", func() { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + <-release + })) + defer server.Close() + defer close(release) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + client := NewClient[[]byte](Args{CookieJar: jar}) + _, err := client.Send(Request{Context: ctx, URL: server.URL, Method: MethodGET, ResponseFormat: ResponseFormatRaw}) + Expect(errors.Is(err, context.DeadlineExceeded)).To(BeTrue()) + }) + + It("does not send a canceled request", func() { + var visited atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { visited.Store(true) })) + defer server.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := NewClient[[]byte](Args{CookieJar: jar}) + _, err := client.Send(Request{Context: ctx, URL: server.URL, Method: MethodGET, ResponseFormat: ResponseFormatRaw}) + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + Expect(visited.Load()).To(BeFalse()) + }) +})