(NOBIDS) migrate apple api to v2, use old one to resolve legacy purch…

…ases, migrator for all purchases

.gitignore

@@ -32,3 +32,4 @@ _gitignore/
 local-deployment/config.yml
 local-deployment/elconfig.json
 local-deployment/.env
+__gitignore

cmd/misc/main.go

@@ -3,7 +3,9 @@ package main
 import (
 	"bytes"
 	"context"
+
 	"database/sql"
+	"encoding/base64"
 	"encoding/json"
 	"eth2-exporter/cmd/misc/commands"
 	"eth2-exporter/db"
@@ -32,6 +34,8 @@ import (
 
 	"flag"
 
+	"github.com/Gurpartap/storekit-go"
+
 	"github.com/sirupsen/logrus"
 )
 
@@ -62,7 +66,7 @@ func main() {
 	statsPartitionCommand := commands.StatsMigratorCommand{}
 
 	configPath := flag.String("config", "config/default.config.yml", "Path to the config file")
-	flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats")
+	flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases")
 	flag.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch")
 	flag.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch")
 	flag.Uint64Var(&opts.User, "user", 0, "user id")
@@ -273,6 +277,8 @@ func main() {
 		indexMissingBlocks(opts.StartBlock, opts.EndBlock, bt, erigonClient)
 	case "migrate-last-attestation-slot-bigtable":
 		migrateLastAttestationSlotToBigtable()
+	case "migrate-app-purchases":
+		err = migrateAppPurchases(opts.Key)
 	case "export-genesis-validators":
 		logrus.Infof("retrieving genesis validator state")
 		validators, err := rpcClient.GetValidatorState(0)
@@ -635,6 +641,135 @@ func fixEnsAddresses(erigonClient *rpc.ErigonClient) error {
 	return nil
 }
 
+func migrateAppPurchases(appStoreSecret string) error {
+	// This code runs once so please don't judge code style too harshly
+
+	if appStoreSecret == "" {
+		return fmt.Errorf("appStoreSecret is empty")
+	}
+
+	client := storekit.NewVerificationClient().OnProductionEnv()
+
+	// Delete marked as duplicate, though the duplicate reject reason is not always set - mainly missing on historical data
+	_, err := db.WriterDb.Exec("DELETE FROM users_app_subscriptions WHERE store = 'ios-appstore' AND reject_reason = 'duplicate';")
+	if err != nil {
+		return errors.Wrap(err, "error deleting duplicate receipt")
+	}
+
+	// Backup legacy receipts into custom column
+	_, err = db.WriterDb.Exec("UPDATE users_app_subscriptions set legacy_receipt = receipt where legacy_receipt is null;")
+	if err != nil {
+		return errors.Wrap(err, "error backing up legacy receipts")
+	}
+
+	receipts := []*types.PremiumData{}
+	err = db.WriterDb.Select(&receipts,
+		"SELECT id, receipt, store, active, validate_remotely, expires_at, product_id, user_id from users_app_subscriptions order by id desc",
+	)
+	if err != nil {
+		return errors.Wrap(err, "error getting app subscriptions")
+	}
+
+	for _, receipt := range receipts {
+		if receipt.Store != "ios-appstore" { // only interested in migrating iOS
+			continue
+		}
+		if len(receipt.Receipt) < 100 { // dont migrate data that has already been migrated (new receipt is a number of a hand full of digits while old one is insanely large)
+			continue
+		}
+
+		receiptData, err := base64.StdEncoding.DecodeString(receipt.Receipt)
+		if err != nil {
+			return errors.Wrap(err, "error decoding receipt")
+		}
+
+		// Call old deprecated endpoint to get the origin transaction id (new receipt info for new endpoints)
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		_, resp, err := client.Verify(ctx, &storekit.ReceiptRequest{
+			ReceiptData:            receiptData,
+			Password:               appStoreSecret,
+			ExcludeOldTransactions: true,
+		})
+
+		if err != nil {
+			return errors.Wrap(err, "error verifying receipt")
+		}
+
+		if resp.LatestReceiptInfo == nil || len(resp.LatestReceiptInfo) == 0 {
+			logrus.Infof("no receipt info for purchase id %v", receipt.ID)
+			if receipt.Active && receipt.ValidateRemotely { // sanity, if there is an active subscription without receipt info we cam't delete it.
+				return fmt.Errorf("no receipt info for active purchase id %v", receipt.ID)
+			}
+			// since it is not active any more and we don't get any new receipt info from apple, just drop the receipt info
+			// hash can stay the same since a collision is unlikely (new and old receipt info)
+			_, err = db.WriterDb.Exec("UPDATE users_app_subscriptions SET receipt = '' WHERE id = $1", receipt.ID)
+			if err != nil {
+				return errors.Wrap(err, "error deleting duplicate receipt")
+			}
+			continue
+		}
+
+		latestReceiptInfo := resp.LatestReceiptInfo[0]
+		logrus.Infof("Update purchase id %v with new receipt %v", receipt.ID, latestReceiptInfo.OriginalTransactionId)
+
+		_, err = db.WriterDb.Exec("UPDATE users_app_subscriptions SET receipt = $1, receipt_hash = $2 WHERE id = $3", latestReceiptInfo.OriginalTransactionId, utils.HashAndEncode(latestReceiptInfo.OriginalTransactionId), receipt.ID)
+		if err != nil {
+			if strings.Contains(err.Error(), "duplicate key") { // handle historic duplicates
+				// get the duplicate receipt
+				duplicateReceipt := types.PremiumData{}
+				err = db.WriterDb.Get(&duplicateReceipt, "SELECT id, user_id, active FROM users_app_subscriptions WHERE receipt_hash = $1", utils.HashAndEncode(latestReceiptInfo.OriginalTransactionId))
+				if err != nil {
+					return errors.Wrap(err, "error getting duplicate receipt")
+				}
+
+				// Keep the active receipt and delete the other one. In case both are inactive keep the newest
+				var deleteReceiptID uint64
+				if !duplicateReceipt.Active && receipt.Active {
+					deleteReceiptID = duplicateReceipt.ID
+				} else if duplicateReceipt.Active && !receipt.Active {
+					deleteReceiptID = receipt.ID
+				} else if !duplicateReceipt.Active && !receipt.Active {
+					if duplicateReceipt.ID > receipt.ID { // keep the newer one
+						deleteReceiptID = duplicateReceipt.ID
+					} else {
+						deleteReceiptID = receipt.ID
+					}
+				} else {
+					return fmt.Errorf("duplicate receipt has same active status: %v != %v for id: %v != %v", duplicateReceipt.Active, receipt.Active, duplicateReceipt.ID, receipt.ID)
+				}
+
+				// new ios handler will automatically update the product id if the user switched the package, so we will just drop this receipt
+				_, err = db.WriterDb.Exec("DELETE FROM users_app_subscriptions WHERE id = $1", deleteReceiptID)
+				if err != nil {
+					return errors.Wrap(err, "error deleting duplicate receipt")
+				}
+				logrus.Infof("deleted duplicate receipt id %v", receipt.ID)
+
+				// the one we keep and update is opposite of the one we deleted
+				var updateReceiptID uint64
+				if deleteReceiptID == duplicateReceipt.ID {
+					updateReceiptID = receipt.ID
+				} else {
+					updateReceiptID = duplicateReceipt.ID
+				}
+
+				_, err = db.WriterDb.Exec("UPDATE users_app_subscriptions SET receipt = $1, receipt_hash = $2 WHERE id = $3", latestReceiptInfo.OriginalTransactionId, utils.HashAndEncode(latestReceiptInfo.OriginalTransactionId), updateReceiptID)
+				if err != nil {
+					return errors.Wrap(err, "error updating receipt")
+				}
+			} else {
+				return errors.Wrap(err, "error updating purchase id")
+			}
+		}
+		logrus.Infof("Migrated purchase id %v\n", receipt.ID)
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	logrus.Infof("done migrating data")
+	return nil
+}
+
 func fixExecTransactionsCount() error {
 	startBlockNumber := uint64(opts.StartBlock)
 	endBlockNumber := uint64(opts.EndBlock)

db/frontend.go

@@ -405,12 +405,12 @@ func InsertMobileSubscription(tx *sql.Tx, userID uint64, paymentDetails types.Mo
 	var err error
 	if tx == nil {
 		_, err = FrontendWriterDB.Exec("INSERT INTO users_app_subscriptions (user_id, product_id, price_micros, currency, created_at, updated_at, validate_remotely, active, store, receipt, expires_at, reject_reason, receipt_hash, subscription_id) VALUES("+
-			"$1, $2, $3, $4, TO_TIMESTAMP($5), TO_TIMESTAMP($6), $7, $8, $9, $10, TO_TIMESTAMP($11), $12, $13, $14);",
+			"$1, $2, $3, $4, TO_TIMESTAMP($5), TO_TIMESTAMP($6), $7, $8, $9, $10, TO_TIMESTAMP($11), $12, $13, $14) ON CONFLICT(receipt_hash) DO UPDATE SET product_id = $2, active = $7, updated_at = TO_TIMESTAMP($5);",
 			userID, paymentDetails.ProductID, paymentDetails.PriceMicros, paymentDetails.Currency, nowTs, nowTs, paymentDetails.Valid, paymentDetails.Valid, store, receipt, expiration, rejectReson, receiptHash, extSubscriptionId,
 		)
 	} else {
 		_, err = tx.Exec("INSERT INTO users_app_subscriptions (user_id, product_id, price_micros, currency, created_at, updated_at, validate_remotely, active, store, receipt, expires_at, reject_reason, receipt_hash, subscription_id) VALUES("+
-			"$1, $2, $3, $4, TO_TIMESTAMP($5), TO_TIMESTAMP($6), $7, $8, $9, $10, TO_TIMESTAMP($11), $12, $13, $14);",
+			"$1, $2, $3, $4, TO_TIMESTAMP($5), TO_TIMESTAMP($6), $7, $8, $9, $10, TO_TIMESTAMP($11), $12, $13, $14) ON CONFLICT(receipt_hash) DO UPDATE SET product_id = $2, active = $7, updated_at = TO_TIMESTAMP($5);",
 			userID, paymentDetails.ProductID, paymentDetails.PriceMicros, paymentDetails.Currency, nowTs, nowTs, paymentDetails.Valid, paymentDetails.Valid, store, receipt, expiration, rejectReson, receiptHash, extSubscriptionId,
 		)
 	}
@@ -484,7 +484,7 @@ func GetAllAppSubscriptions() ([]*types.PremiumData, error) {
 	data := []*types.PremiumData{}
 
 	err := FrontendWriterDB.Select(&data,
-		"SELECT id, receipt, store, active, expires_at, product_id from users_app_subscriptions WHERE validate_remotely = true order by id desc",
+		"SELECT id, receipt, store, active, expires_at, product_id, user_id, validate_remotely from users_app_subscriptions WHERE validate_remotely = true order by id desc",
 	)
 
 	return data, err
@@ -531,6 +531,21 @@ func UpdateUserSubscription(tx *sql.Tx, id uint64, valid bool, expiration int64,
 	return err
 }
 
+func UpdateUserSubscriptionProduct(tx *sql.Tx, id uint64, productID string) error {
+	var err error
+	if tx == nil {
+		_, err = FrontendWriterDB.Exec("UPDATE users_app_subscriptions SET product_id = $1 WHERE id = $2;",
+			productID, id,
+		)
+	} else {
+		_, err = tx.Exec("UPDATE users_app_subscriptions SET product_id = $1 WHERE id = $2",
+			productID, id,
+		)
+	}
+
+	return err
+}
+
 func SetSubscriptionToExpired(tx *sql.Tx, id uint64) error {
 	var err error
 	query := "UPDATE users_app_subscriptions SET validate_remotely = false, reject_reason = 'expired' WHERE id = $1;"

db/migrations/20240108133931_app_subscriptions_id.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE users_app_subscriptions ADD COLUMN legacy_receipt varchar(150000);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE users_app_subscriptions DROP COLUMN legacy_receipt;
+-- +goose StatementEnd

db/migrations/new.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Ask for name of migration
+echo "Enter name of migration file (for example add_validators_indices): "
+read -r name
+
+# This script creates a new migration file with the current timestamp
+# as the filename prefix.
+filename=$(date +"%Y%m%d%H%M%S")_$name.sql
+touch $filename
+
+cat <<EOF > $filename
+-- +goose Up
+-- +goose StatementBegin
+
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+
+-- +goose StatementEnd
+EOF
+
+