From df9901102cb58ee8024163baadc8b7a2f1fe25bc Mon Sep 17 00:00:00 2001 From: startupcompliancekit <331868586+startupcompliancekit@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:39:19 -0500 Subject: [PATCH] Preserve raw token and NFT metadata in PostgreSQL bytea --- README.md | 16 ++ .../explorer/persistence/Migrations.scala | 33 +++- .../model/FungibleTokenMetadataEntity.scala | 21 +++ .../model/NFTCollectionMetadataEntity.scala | 17 ++ .../persistence/model/NFTMetadataEntity.scala | 20 +++ .../persistence/queries/ContractQueries.scala | 8 +- .../persistence/queries/TokenQueries.scala | 23 +-- .../persistence/schema/CustomGetResult.scala | 12 +- .../schema/FungibleTokenMetadataSchema.scala | 19 +-- .../schema/NFTCollectionMetadataSchema.scala | 13 +- .../schema/NFTMetadataSchema.scala | 13 +- .../explorer/service/BlockFlowClient.scala | 45 +++--- .../explorer/service/TokenService.scala | 6 +- .../explorer/service/TransactionService.scala | 10 +- .../persistence/MetadataBytesSpec.scala | 147 ++++++++++++++++++ .../queries/TokenQueriesSpec.scala | 31 +++- .../service/BlockFlowClientSpec.scala | 18 +-- .../service/EmptyBlockFlowClient.scala | 8 +- 18 files changed, 374 insertions(+), 86 deletions(-) create mode 100644 app/src/main/scala/org/alephium/explorer/persistence/model/FungibleTokenMetadataEntity.scala create mode 100644 app/src/main/scala/org/alephium/explorer/persistence/model/NFTCollectionMetadataEntity.scala create mode 100644 app/src/main/scala/org/alephium/explorer/persistence/model/NFTMetadataEntity.scala create mode 100644 app/src/test/scala/org/alephium/explorer/persistence/MetadataBytesSpec.scala diff --git a/README.md b/README.md index a72c67a4f..4e7483a41 100644 --- a/README.md +++ b/README.md @@ -204,3 +204,19 @@ apiMappings ++= ) ) ``` + +### Token metadata storage migration + +Database migration 12 stores fungible token symbols and names, NFT token URIs, +and NFT collection URIs as PostgreSQL `bytea`. Startup converts existing text +columns using UTF-8 encoding; literal backslashes remain literal data. Columns +already stored as `bytea` are left unchanged, including on migration replay. +The migration runs in the normal migration transaction and may lock and rewrite +the affected tables. Schedule upgrades with their size in mind. + +New metadata is persisted exactly as returned by contract calls, including NUL +bytes and malformed UTF-8. The API still returns the same string fields: UTF-8 +decoding happens when producing API models, with replacement characters for +malformed sequences. NUL characters are retained and escaped by JSON encoding. +The earlier metadata conversion stripped NUL characters; migration cannot recover +bytes already removed or replaced in existing text records. diff --git a/app/src/main/scala/org/alephium/explorer/persistence/Migrations.scala b/app/src/main/scala/org/alephium/explorer/persistence/Migrations.scala index ad29e502d..dd1bdf05f 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/Migrations.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/Migrations.scala @@ -29,7 +29,7 @@ import org.alephium.util.discard object Migrations extends StrictLogging { // scalastyle:off magic.number - val latestVersion: MigrationVersion = MigrationVersion(11) + val latestVersion: MigrationVersion = MigrationVersion(12) def migration1(implicit ec: ExecutionContext): DBActionAll[Unit] = { // We retrigger the download of fungible and non-fungible tokens' metadata that have sub-category @@ -222,6 +222,34 @@ object Migrations extends StrictLogging { """.map(_ => ()) } + // Existing text is encoded as UTF-8, not interpreted as PostgreSQL bytea input syntax. + // The migration can be replayed after a reset migration counter and on fresh bytea schemas. + def migration12(implicit ec: ExecutionContext): DBActionAll[Unit] = { + DBIO.seq( + metadataColumnToBytes("fungible_token_metadata", "symbol"), + metadataColumnToBytes("fungible_token_metadata", "name"), + metadataColumnToBytes("nft_metadata", "token_uri"), + metadataColumnToBytes("nft_collection_metadata", "collection_uri") + ) + } + + private def metadataColumnToBytes(table: String, column: String)(implicit + ec: ExecutionContext + ): DBActionAll[Unit] = { + sql"""SELECT data_type FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $table AND column_name = $column""" + .as[String] + .head + .flatMap { + case "bytea" => DBIO.successful(()) + case "text" | "character varying" | "character" => + sqlu"""ALTER TABLE #$table ALTER COLUMN #$column TYPE bytea + USING convert_to(#$column, 'UTF8')""".map(_ => ()) + case other => + DBIO.failed(new IllegalStateException(s"Unexpected metadata column type: $other")) + } + } + private def migrations(implicit explorerConfig: ExplorerConfig, ec: ExecutionContext @@ -236,7 +264,8 @@ object Migrations extends StrictLogging { migration8, migration9, migration10, - migration11 + migration11, + migration12 ) def backgroundCoinbaseMigration()(implicit diff --git a/app/src/main/scala/org/alephium/explorer/persistence/model/FungibleTokenMetadataEntity.scala b/app/src/main/scala/org/alephium/explorer/persistence/model/FungibleTokenMetadataEntity.scala new file mode 100644 index 000000000..c0ef831c3 --- /dev/null +++ b/app/src/main/scala/org/alephium/explorer/persistence/model/FungibleTokenMetadataEntity.scala @@ -0,0 +1,21 @@ +// Copyright (c) Alephium +// SPDX-License-Identifier: LGPL-3.0-only + +package org.alephium.explorer.persistence.model + +import org.apache.pekko.util.ByteString + +import org.alephium.explorer.api.model.FungibleTokenMetadata +import org.alephium.protocol.model.TokenId +import org.alephium.util.U256 + +// Preserve arbitrary contract bytes until constructing the public API representation. +final case class FungibleTokenMetadataEntity( + id: TokenId, + symbol: ByteString, + name: ByteString, + decimals: U256 +) { + def toApi(): FungibleTokenMetadata = + FungibleTokenMetadata(id, symbol.utf8String, name.utf8String, decimals) +} diff --git a/app/src/main/scala/org/alephium/explorer/persistence/model/NFTCollectionMetadataEntity.scala b/app/src/main/scala/org/alephium/explorer/persistence/model/NFTCollectionMetadataEntity.scala new file mode 100644 index 000000000..990f16001 --- /dev/null +++ b/app/src/main/scala/org/alephium/explorer/persistence/model/NFTCollectionMetadataEntity.scala @@ -0,0 +1,17 @@ +// Copyright (c) Alephium +// SPDX-License-Identifier: LGPL-3.0-only + +package org.alephium.explorer.persistence.model + +import org.apache.pekko.util.ByteString + +import org.alephium.explorer.api.model.NFTCollectionMetadata +import org.alephium.protocol.model.Address + +// Preserve arbitrary contract bytes until constructing the public API representation. +final case class NFTCollectionMetadataEntity( + address: Address.Contract, + collectionUri: ByteString +) { + def toApi(): NFTCollectionMetadata = NFTCollectionMetadata(address, collectionUri.utf8String) +} diff --git a/app/src/main/scala/org/alephium/explorer/persistence/model/NFTMetadataEntity.scala b/app/src/main/scala/org/alephium/explorer/persistence/model/NFTMetadataEntity.scala new file mode 100644 index 000000000..af10425c9 --- /dev/null +++ b/app/src/main/scala/org/alephium/explorer/persistence/model/NFTMetadataEntity.scala @@ -0,0 +1,20 @@ +// Copyright (c) Alephium +// SPDX-License-Identifier: LGPL-3.0-only + +package org.alephium.explorer.persistence.model + +import org.apache.pekko.util.ByteString + +import org.alephium.explorer.api.model.NFTMetadata +import org.alephium.protocol.model.{ContractId, TokenId} +import org.alephium.util.U256 + +// Preserve arbitrary contract bytes until constructing the public API representation. +final case class NFTMetadataEntity( + id: TokenId, + tokenUri: ByteString, + collectionId: ContractId, + nftIndex: U256 +) { + def toApi(): NFTMetadata = NFTMetadata(id, tokenUri.utf8String, collectionId, nftIndex) +} diff --git a/app/src/main/scala/org/alephium/explorer/persistence/queries/ContractQueries.scala b/app/src/main/scala/org/alephium/explorer/persistence/queries/ContractQueries.scala index 03f3b45a5..98392a2b8 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/queries/ContractQueries.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/queries/ContractQueries.scala @@ -11,7 +11,11 @@ import slick.jdbc.PostgresProfile.api._ import org.alephium.explorer.api.model._ import org.alephium.explorer.persistence._ -import org.alephium.explorer.persistence.model.{ContractEntity, EventEntity} +import org.alephium.explorer.persistence.model.{ + ContractEntity, + EventEntity, + NFTCollectionMetadataEntity +} import org.alephium.explorer.persistence.schema.CustomGetResult._ import org.alephium.explorer.persistence.schema.CustomSetParameter._ import org.alephium.explorer.util.SlickUtil._ @@ -160,7 +164,7 @@ object ContractQueries { def insertNFTCollectionMetadata( contract: Address, - metadata: NFTCollectionMetadata + metadata: NFTCollectionMetadataEntity ): DBActionW[Int] = { sqlu""" INSERT INTO nft_collection_metadata ( diff --git a/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala b/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala index 56e5baaaa..19e706215 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala @@ -14,7 +14,12 @@ import slick.jdbc.PostgresProfile.api._ import org.alephium.api.model.{Address => ApiAddress} import org.alephium.explorer.api.model._ import org.alephium.explorer.persistence._ -import org.alephium.explorer.persistence.model.TokenInfoEntity +import org.alephium.explorer.persistence.model.{ + FungibleTokenMetadataEntity, + NFTCollectionMetadataEntity, + NFTMetadataEntity, + TokenInfoEntity +} import org.alephium.explorer.persistence.queries.result.TxByTokenQR import org.alephium.explorer.persistence.schema.CustomGetResult._ import org.alephium.explorer.persistence.schema.CustomSetParameter._ @@ -221,7 +226,7 @@ object TokenQueries extends StrictLogging { .asAS[(TokenId, Option[U256], Option[U256])] def insertFungibleTokenMetadata( - metadata: FungibleTokenMetadata + metadata: FungibleTokenMetadataEntity ): DBActionW[Int] = { sqlu""" INSERT INTO fungible_token_metadata ( @@ -235,7 +240,7 @@ object TokenQueries extends StrictLogging { def listFungibleTokenMetadataQuery( tokens: ArraySeq[TokenId] - ): DBActionW[ArraySeq[FungibleTokenMetadata]] = { + ): DBActionW[ArraySeq[FungibleTokenMetadataEntity]] = { if (tokens.nonEmpty) { val params = paramPlaceholder(1, tokens.size) @@ -255,7 +260,7 @@ object TokenQueries extends StrictLogging { SQLActionBuilder( sql = query, setParameter = parameters - ).asASE[FungibleTokenMetadata](fungibleTokenMetadataGetResult) + ).asASE[FungibleTokenMetadataEntity](fungibleTokenMetadataGetResult) } else { DBIOAction.successful(ArraySeq.empty) } @@ -291,7 +296,7 @@ object TokenQueries extends StrictLogging { def listNFTMetadataQuery( tokens: ArraySeq[TokenId] - ): DBActionW[ArraySeq[NFTMetadata]] = { + ): DBActionW[ArraySeq[NFTMetadataEntity]] = { if (tokens.nonEmpty) { val params = paramPlaceholder(1, tokens.size) @@ -311,7 +316,7 @@ object TokenQueries extends StrictLogging { SQLActionBuilder( sql = query, setParameter = parameters - ).asASE[NFTMetadata](nftMetadataGetResult) + ).asASE[NFTMetadataEntity](nftMetadataGetResult) } else { DBIOAction.successful(ArraySeq.empty) } @@ -319,7 +324,7 @@ object TokenQueries extends StrictLogging { def listNFTCollectionMetadataQuery( addresses: ArraySeq[Address.Contract] - ): DBActionW[ArraySeq[NFTCollectionMetadata]] = { + ): DBActionW[ArraySeq[NFTCollectionMetadataEntity]] = { if (addresses.nonEmpty) { val params = paramPlaceholder(1, addresses.size) @@ -339,14 +344,14 @@ object TokenQueries extends StrictLogging { SQLActionBuilder( sql = query, setParameter = parameters - ).asASE[NFTCollectionMetadata](nftCollectionMetadataGetResult) + ).asASE[NFTCollectionMetadataEntity](nftCollectionMetadataGetResult) } else { DBIOAction.successful(ArraySeq.empty) } } def insertNFTMetadata( - metadata: NFTMetadata + metadata: NFTMetadataEntity ): DBActionW[Int] = { sqlu""" INSERT INTO nft_metadata ( diff --git a/app/src/main/scala/org/alephium/explorer/persistence/schema/CustomGetResult.scala b/app/src/main/scala/org/alephium/explorer/persistence/schema/CustomGetResult.scala index 7caed4c44..705d6262a 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/schema/CustomGetResult.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/schema/CustomGetResult.scala @@ -398,27 +398,27 @@ object CustomGetResult { eventOrder = result.<< ) - val fungibleTokenMetadataGetResult: GetResult[FungibleTokenMetadata] = + val fungibleTokenMetadataGetResult: GetResult[FungibleTokenMetadataEntity] = (result: PositionedResult) => - FungibleTokenMetadata( + FungibleTokenMetadataEntity( id = result.<<, symbol = result.<<, name = result.<<, decimals = result.<< ) - val nftMetadataGetResult: GetResult[NFTMetadata] = + val nftMetadataGetResult: GetResult[NFTMetadataEntity] = (result: PositionedResult) => - NFTMetadata( + NFTMetadataEntity( id = result.<<, tokenUri = result.<<, collectionId = result.<<, nftIndex = result.<< ) - val nftCollectionMetadataGetResult: GetResult[NFTCollectionMetadata] = + val nftCollectionMetadataGetResult: GetResult[NFTCollectionMetadataEntity] = (result: PositionedResult) => - NFTCollectionMetadata( + NFTCollectionMetadataEntity( address = result.<<(addressContractGetResult), collectionUri = result.<< ) diff --git a/app/src/main/scala/org/alephium/explorer/persistence/schema/FungibleTokenMetadataSchema.scala b/app/src/main/scala/org/alephium/explorer/persistence/schema/FungibleTokenMetadataSchema.scala index ef6a18289..d82cc672f 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/schema/FungibleTokenMetadataSchema.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/schema/FungibleTokenMetadataSchema.scala @@ -3,26 +3,27 @@ package org.alephium.explorer.persistence.schema +import org.apache.pekko.util.ByteString import slick.jdbc.PostgresProfile.api._ import slick.lifted.ProvenShape -import org.alephium.explorer.api.model.FungibleTokenMetadata +import org.alephium.explorer.persistence.model.FungibleTokenMetadataEntity import org.alephium.explorer.persistence.schema.CustomJdbcTypes._ import org.alephium.protocol.model.TokenId import org.alephium.util.U256 object FungibleTokenMetadataSchema - extends SchemaMainChain[FungibleTokenMetadata]("fungible_token_metadata") { + extends SchemaMainChain[FungibleTokenMetadataEntity]("fungible_token_metadata") { - class TokenInfos(tag: Tag) extends Table[FungibleTokenMetadata](tag, name) { - def token: Rep[TokenId] = column[TokenId]("token", O.PrimaryKey) - def symbol: Rep[String] = column[String]("symbol") - def name: Rep[String] = column[String]("name") - def decimals: Rep[U256] = column[U256]("decimals", O.SqlType("DECIMAL(80,0)")) + class TokenInfos(tag: Tag) extends Table[FungibleTokenMetadataEntity](tag, name) { + def token: Rep[TokenId] = column[TokenId]("token", O.PrimaryKey) + def symbol: Rep[ByteString] = column[ByteString]("symbol") + def name: Rep[ByteString] = column[ByteString]("name") + def decimals: Rep[U256] = column[U256]("decimals", O.SqlType("DECIMAL(80,0)")) - def * : ProvenShape[FungibleTokenMetadata] = + def * : ProvenShape[FungibleTokenMetadataEntity] = (token, symbol, name, decimals) - .<>((FungibleTokenMetadata.apply _).tupled, FungibleTokenMetadata.unapply) + .<>((FungibleTokenMetadataEntity.apply _).tupled, FungibleTokenMetadataEntity.unapply) } val table: TableQuery[TokenInfos] = TableQuery[TokenInfos] diff --git a/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTCollectionMetadataSchema.scala b/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTCollectionMetadataSchema.scala index c8a621640..e325fd7b6 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTCollectionMetadataSchema.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTCollectionMetadataSchema.scala @@ -3,23 +3,24 @@ package org.alephium.explorer.persistence.schema +import org.apache.pekko.util.ByteString import slick.jdbc.PostgresProfile.api._ import slick.lifted.ProvenShape -import org.alephium.explorer.api.model.NFTCollectionMetadata +import org.alephium.explorer.persistence.model.NFTCollectionMetadataEntity import org.alephium.explorer.persistence.schema.CustomJdbcTypes._ import org.alephium.protocol.model.Address object NFTCollectionMetadataSchema - extends SchemaMainChain[NFTCollectionMetadata]("nft_collection_metadata") { + extends SchemaMainChain[NFTCollectionMetadataEntity]("nft_collection_metadata") { - class TokenInfos(tag: Tag) extends Table[NFTCollectionMetadata](tag, name) { + class TokenInfos(tag: Tag) extends Table[NFTCollectionMetadataEntity](tag, name) { def contract: Rep[Address.Contract] = column[Address.Contract]("contract", O.PrimaryKey) - def collectionUri: Rep[String] = column[String]("collection_uri") + def collectionUri: Rep[ByteString] = column[ByteString]("collection_uri") - def * : ProvenShape[NFTCollectionMetadata] = + def * : ProvenShape[NFTCollectionMetadataEntity] = (contract, collectionUri) - .<>((NFTCollectionMetadata.apply _).tupled, NFTCollectionMetadata.unapply) + .<>((NFTCollectionMetadataEntity.apply _).tupled, NFTCollectionMetadataEntity.unapply) } val table: TableQuery[TokenInfos] = TableQuery[TokenInfos] diff --git a/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTMetadataSchema.scala b/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTMetadataSchema.scala index e3d3e8c94..e6bc0083c 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTMetadataSchema.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/schema/NFTMetadataSchema.scala @@ -3,25 +3,26 @@ package org.alephium.explorer.persistence.schema +import org.apache.pekko.util.ByteString import slick.jdbc.PostgresProfile.api._ import slick.lifted.ProvenShape -import org.alephium.explorer.api.model.NFTMetadata +import org.alephium.explorer.persistence.model.NFTMetadataEntity import org.alephium.explorer.persistence.schema.CustomJdbcTypes._ import org.alephium.protocol.model.{ContractId, TokenId} import org.alephium.util.U256 -object NFTMetadataSchema extends SchemaMainChain[NFTMetadata]("nft_metadata") { +object NFTMetadataSchema extends SchemaMainChain[NFTMetadataEntity]("nft_metadata") { - class NFTMetadatas(tag: Tag) extends Table[NFTMetadata](tag, name) { + class NFTMetadatas(tag: Tag) extends Table[NFTMetadataEntity](tag, name) { def token: Rep[TokenId] = column[TokenId]("token", O.PrimaryKey) - def tokenUri: Rep[String] = column[String]("token_uri") + def tokenUri: Rep[ByteString] = column[ByteString]("token_uri") def collectionId: Rep[ContractId] = column[ContractId]("collection_id") def nftIndex: Rep[U256] = column[U256]("nft_index", O.SqlType("DECIMAL(80,0)")) - def * : ProvenShape[NFTMetadata] = + def * : ProvenShape[NFTMetadataEntity] = (token, tokenUri, collectionId, nftIndex) - .<>((NFTMetadata.apply _).tupled, NFTMetadata.unapply) + .<>((NFTMetadataEntity.apply _).tupled, NFTMetadataEntity.unapply) } val table: TableQuery[NFTMetadatas] = TableQuery[NFTMetadatas] diff --git a/app/src/main/scala/org/alephium/explorer/service/BlockFlowClient.scala b/app/src/main/scala/org/alephium/explorer/service/BlockFlowClient.scala index 2564b4e9e..a9e98ede8 100644 --- a/app/src/main/scala/org/alephium/explorer/service/BlockFlowClient.scala +++ b/app/src/main/scala/org/alephium/explorer/service/BlockFlowClient.scala @@ -90,11 +90,13 @@ trait BlockFlowClient extends Service { def guessTokenStdInterfaceId(token: TokenId): Future[Option[StdInterfaceId]] - def fetchFungibleTokenMetadata(token: TokenId): Future[Option[FungibleTokenMetadata]] + def fetchFungibleTokenMetadata(token: TokenId): Future[Option[FungibleTokenMetadataEntity]] - def fetchNFTMetadata(token: TokenId): Future[Option[NFTMetadata]] + def fetchNFTMetadata(token: TokenId): Future[Option[NFTMetadataEntity]] - def fetchNFTCollectionMetadata(contract: Address.Contract): Future[Option[NFTCollectionMetadata]] + def fetchNFTCollectionMetadata( + contract: Address.Contract + ): Future[Option[NFTCollectionMetadataEntity]] def start(): Future[Unit] @@ -300,22 +302,22 @@ object BlockFlowClient extends StrictLogging { } } - def fetchFungibleTokenMetadata(token: TokenId): Future[Option[FungibleTokenMetadata]] = { - fetchTokenMetadata[FungibleTokenMetadata](token, 3) { result => + def fetchFungibleTokenMetadata(token: TokenId): Future[Option[FungibleTokenMetadataEntity]] = { + fetchTokenMetadata[FungibleTokenMetadataEntity](token, 3) { result => extractFungibleTokenMetadata(token, result) } } def fetchNFTCollectionMetadata( contract: Address.Contract - ): Future[Option[NFTCollectionMetadata]] = { - fetchMetadata[NFTCollectionMetadata](contract, 1) { result => + ): Future[Option[NFTCollectionMetadataEntity]] = { + fetchMetadata[NFTCollectionMetadataEntity](contract, 1) { result => extractNFTCollectionMetadata(contract, result) } } - def fetchNFTMetadata(token: TokenId): Future[Option[NFTMetadata]] = { - fetchTokenMetadata[NFTMetadata](token, 2) { result => + def fetchNFTMetadata(token: TokenId): Future[Option[NFTMetadataEntity]] = { + fetchTokenMetadata[NFTMetadataEntity](token, 2) { result => extractNFTMetadata(token, result) } } @@ -363,6 +365,11 @@ object BlockFlowClient extends StrictLogging { case _ => None } + def valToBytes(value: api.model.Val): Option[ByteString] = value match { + case api.model.ValByteVec(bytes) => Some(bytes) + case _ => None + } + def valToString(value: api.model.Val): Option[String] = value match { case api.model.ValByteVec(bytes) => @@ -391,13 +398,13 @@ object BlockFlowClient extends StrictLogging { def extractFungibleTokenMetadata( token: TokenId, result: MultipleCallContractResult - ): Option[FungibleTokenMetadata] = { + ): Option[FungibleTokenMetadataEntity] = { for { - symbol <- extractVal(result.results(0)).flatMap(valToString) - name <- extractVal(result.results(1)).flatMap(valToString) + symbol <- extractVal(result.results(0)).flatMap(valToBytes) + name <- extractVal(result.results(1)).flatMap(valToBytes) decimals <- extractVal(result.results(2)).flatMap(valToU256) } yield { - FungibleTokenMetadata( + FungibleTokenMetadataEntity( token, symbol, name, @@ -409,11 +416,11 @@ object BlockFlowClient extends StrictLogging { def extractNFTCollectionMetadata( contract: Address.Contract, result: MultipleCallContractResult - ): Option[NFTCollectionMetadata] = { + ): Option[NFTCollectionMetadataEntity] = { for { - collectionUri <- extractVal(result.results(0)).flatMap(valToString) + collectionUri <- extractVal(result.results(0)).flatMap(valToBytes) } yield { - NFTCollectionMetadata( + NFTCollectionMetadataEntity( contract, collectionUri ) @@ -422,13 +429,13 @@ object BlockFlowClient extends StrictLogging { def extractNFTMetadata( token: TokenId, result: MultipleCallContractResult - ): Option[NFTMetadata] = { + ): Option[NFTMetadataEntity] = { for { - tokenUri <- extractVal(result.results(0)).flatMap(valToString) + tokenUri <- extractVal(result.results(0)).flatMap(valToBytes) collectionId <- extractVal(result.results(1)).flatMap(valToContractId) nftIndex <- extractVal(result.results(1), _.tail.headOption).flatMap(valToU256) } yield { - NFTMetadata( + NFTMetadataEntity( token, tokenUri, collectionId, diff --git a/app/src/main/scala/org/alephium/explorer/service/TokenService.scala b/app/src/main/scala/org/alephium/explorer/service/TokenService.scala index 938e28e5b..34c30a439 100644 --- a/app/src/main/scala/org/alephium/explorer/service/TokenService.scala +++ b/app/src/main/scala/org/alephium/explorer/service/TokenService.scala @@ -135,17 +135,17 @@ object TokenService extends TokenService with StrictLogging { def listFungibleTokenMetadata(tokens: ArraySeq[TokenId])(implicit dc: DatabaseConfig[PostgresProfile] ): Future[ArraySeq[FungibleTokenMetadata]] = - run(listFungibleTokenMetadataQuery(tokens)) + run(listFungibleTokenMetadataQuery(tokens)).map(_.map(_.toApi()))(ExecutionContext.parasitic) def listNFTMetadata(tokens: ArraySeq[TokenId])(implicit dc: DatabaseConfig[PostgresProfile] ): Future[ArraySeq[NFTMetadata]] = - run(listNFTMetadataQuery(tokens)) + run(listNFTMetadataQuery(tokens)).map(_.map(_.toApi()))(ExecutionContext.parasitic) def listNFTCollectionMetadata(addresses: ArraySeq[Address.Contract])(implicit dc: DatabaseConfig[PostgresProfile] ): Future[ArraySeq[NFTCollectionMetadata]] = - run(listNFTCollectionMetadataQuery(addresses)) + run(listNFTCollectionMetadataQuery(addresses)).map(_.map(_.toApi()))(ExecutionContext.parasitic) def listAddressTokens(address: ApiAddress, pagination: Pagination)(implicit dc: DatabaseConfig[PostgresProfile] diff --git a/app/src/main/scala/org/alephium/explorer/service/TransactionService.scala b/app/src/main/scala/org/alephium/explorer/service/TransactionService.scala index 8870da366..4dbb3c15d 100644 --- a/app/src/main/scala/org/alephium/explorer/service/TransactionService.scala +++ b/app/src/main/scala/org/alephium/explorer/service/TransactionService.scala @@ -23,7 +23,7 @@ import org.alephium.explorer.cache._ import org.alephium.explorer.foldFutures import org.alephium.explorer.persistence.DBRunner._ import org.alephium.explorer.persistence.dao.{MempoolDao, TransactionDao} -import org.alephium.explorer.persistence.queries.{InputQueries, OutputQueries, TokenQueries} +import org.alephium.explorer.persistence.queries.{InputQueries, OutputQueries} import org.alephium.explorer.persistence.queries.TransactionQueries._ import org.alephium.explorer.persistence.queries.result.OutputFromTxQR import org.alephium.protocol @@ -481,9 +481,11 @@ object TransactionService extends TransactionService { val tokens: ArraySeq[TokenId] = transactions.flatMap(_.outputs).flatMap(_.tokens.map(_.map(_.id))).flatten.distinct - run(TokenQueries.listFungibleTokenMetadataQuery(tokens)).map(_.map { metadata => - metadata.id -> metadata - }.toMap) + TokenService + .listFungibleTokenMetadata(tokens) + .map(_.map { metadata => + metadata.id -> metadata + }.toMap) } def transactionsFlowable( diff --git a/app/src/test/scala/org/alephium/explorer/persistence/MetadataBytesSpec.scala b/app/src/test/scala/org/alephium/explorer/persistence/MetadataBytesSpec.scala new file mode 100644 index 000000000..894bf9701 --- /dev/null +++ b/app/src/test/scala/org/alephium/explorer/persistence/MetadataBytesSpec.scala @@ -0,0 +1,147 @@ +// Copyright (c) Alephium +// SPDX-License-Identifier: LGPL-3.0-only + +package org.alephium.explorer.persistence + +import scala.collection.immutable.ArraySeq + +import org.apache.pekko.util.ByteString +import slick.jdbc.PostgresProfile.api._ + +import org.alephium.api.model +import org.alephium.explorer.AlephiumFutureSpec +import org.alephium.explorer.persistence.queries.{ContractQueries, TokenQueries} +import org.alephium.explorer.persistence.schema._ +import org.alephium.explorer.persistence.schema.CustomSetParameter._ +import org.alephium.explorer.service.{BlockFlowClient, TokenService} +import org.alephium.json.Json._ +import org.alephium.protocol.Hash +import org.alephium.protocol.model.{Address, ContractId, TokenId} +import org.alephium.util.{AVector, U256} + +class MetadataBytesSpec extends AlephiumFutureSpec with DatabaseFixtureForEach with TestDBRunner { + private val contractId = ContractId.unsafe(Hash.unsafe(ByteString(Array.fill[Byte](32)(1)))) + private val token = TokenId.from(contractId) + private val address = Address.contract(contractId) + private val malformed = ByteString(Array[Byte](0, -61, 40, -1)) + private val allBytes = ByteString((0 to 255).map(_.toByte).toArray) + + private def contractResult(values: model.Val*): model.CallContractResult = + model.CallContractSucceeded( + AVector.from(values), + 0, + AVector.empty, + AVector.empty, + AVector.empty, + AVector.empty, + AVector.empty + ) + + private def assertByteColumns(): Unit = { + val types = exec( + sql"""SELECT data_type FROM information_schema.columns + WHERE table_schema = current_schema() AND + ((table_name = 'fungible_token_metadata' AND column_name IN ('symbol', 'name')) + OR (table_name = 'nft_metadata' AND column_name = 'token_uri') + OR (table_name = 'nft_collection_metadata' AND column_name = 'collection_uri'))""".as[String] + ) + val _ = types is Vector.fill(4)("bytea") + } + + "Metadata bytes" should { + "round trip arbitrary bytes through SQL and Slick and convert only at the API boundary" in { + val fungible = BlockFlowClient + .extractFungibleTokenMetadata( + token, + model.MultipleCallContractResult( + AVector( + contractResult(model.ValByteVec(malformed)), + contractResult(model.ValByteVec(ByteString("λ token\\x00"))), + contractResult(model.ValU256(U256.Zero)) + ) + ) + ) + .get + val nft = BlockFlowClient + .extractNFTMetadata( + token, + model.MultipleCallContractResult( + AVector( + contractResult(model.ValByteVec(allBytes)), + contractResult(model.ValByteVec(contractId.bytes), model.ValU256(U256.One)) + ) + ) + ) + .get + val collection = BlockFlowClient + .extractNFTCollectionMetadata( + address, + model.MultipleCallContractResult( + AVector(contractResult(model.ValByteVec(ByteString.empty))) + ) + ) + .get + exec(TokenQueries.insertFungibleTokenMetadata(fungible)) + exec(TokenQueries.insertNFTMetadata(nft)) + exec(ContractQueries.insertNFTCollectionMetadata(address, collection)) + assertByteColumns() + exec(TokenQueries.listFungibleTokenMetadataQuery(ArraySeq(token))) is ArraySeq(fungible) + exec(TokenQueries.listNFTMetadataQuery(ArraySeq(token))) is ArraySeq(nft) + exec(TokenQueries.listNFTCollectionMetadataQuery(ArraySeq(address))) is ArraySeq(collection) + exec(FungibleTokenMetadataSchema.table.result) is Vector(fungible) + exec(NFTMetadataSchema.table.result) is Vector(nft) + exec(NFTCollectionMetadataSchema.table.result) is Vector(collection) + + val apiToken = TokenService.listFungibleTokenMetadata(ArraySeq(token)).futureValue.head + apiToken.symbol is "\u0000\ufffd(\ufffd" + apiToken.name is "λ token\\x00" + ujson.read(write(apiToken))("symbol").str is "\u0000\ufffd(\ufffd" + TokenService.listNFTMetadata(ArraySeq(token)).futureValue.head.tokenUri is allBytes.utf8String + TokenService.listNFTCollectionMetadata(ArraySeq(address)).futureValue.head.collectionUri is "" + // API conversion must not replace the stored original with its lossy UTF-8 decoding. + exec(TokenQueries.listNFTMetadataQuery(ArraySeq(token))).head.tokenUri is allBytes + exec(Migrations.migration12.transactionally) + exec(TokenQueries.listFungibleTokenMetadataQuery(ArraySeq(token))).head.symbol is malformed + } + + "migrate legacy Unicode and literal backslashes without interpreting bytea syntax" in { + exec( + sqlu"""ALTER TABLE fungible_token_metadata + ALTER COLUMN symbol TYPE text USING convert_from(symbol, 'UTF8'), + ALTER COLUMN name TYPE text USING convert_from(name, 'UTF8'); + ALTER TABLE nft_metadata ALTER COLUMN token_uri TYPE text USING convert_from(token_uri, 'UTF8'); + ALTER TABLE nft_collection_metadata ALTER COLUMN collection_uri TYPE text USING convert_from(collection_uri, 'UTF8');""" + ) + val symbol = "\\xdeadbeef" + val name = "名字 λ 😀" + val uri = "ipfs://local/\\000/雪" + exec(sqlu"""INSERT INTO fungible_token_metadata VALUES ($token, $symbol, $name, 18)""") + exec(sqlu"""INSERT INTO nft_metadata VALUES ($token, $uri, $contractId, 1)""") + exec(sqlu"""INSERT INTO nft_collection_metadata VALUES ($address, $uri)""") + exec(Migrations.migration12.transactionally) + exec(Migrations.migration12.transactionally) + assertByteColumns() + val ft = exec(TokenQueries.listFungibleTokenMetadataQuery(ArraySeq(token))).head + ft.symbol is ByteString(symbol) + ft.name is ByteString(name) + exec(TokenQueries.listNFTMetadataQuery(ArraySeq(token))).head.tokenUri is ByteString(uri) + exec( + TokenQueries.listNFTCollectionMetadataQuery(ArraySeq(address)) + ).head.collectionUri is ByteString(uri) + TokenService.listFungibleTokenMetadata(ArraySeq(token)).futureValue.head.name is name + } + + "resume a mixed text and bytea schema without rewriting binary values" in { + exec( + sqlu"""ALTER TABLE fungible_token_metadata ALTER COLUMN name TYPE text USING convert_from(name, 'UTF8')""" + ) + val name = "already partially migrated" + exec(sqlu"""INSERT INTO fungible_token_metadata VALUES ($token, $malformed, $name, 0)""") + exec(Migrations.migration12.transactionally) + assertByteColumns() + val result = exec(TokenQueries.listFungibleTokenMetadataQuery(ArraySeq(token))).head + result.symbol is malformed + result.name is ByteString(name) + } + } +} diff --git a/app/src/test/scala/org/alephium/explorer/persistence/queries/TokenQueriesSpec.scala b/app/src/test/scala/org/alephium/explorer/persistence/queries/TokenQueriesSpec.scala index 29a3309d7..1b5822fea 100644 --- a/app/src/test/scala/org/alephium/explorer/persistence/queries/TokenQueriesSpec.scala +++ b/app/src/test/scala/org/alephium/explorer/persistence/queries/TokenQueriesSpec.scala @@ -5,6 +5,7 @@ package org.alephium.explorer.persistence.queries import scala.collection.immutable.ArraySeq +import org.apache.pekko.util.ByteString import org.scalacheck.Gen import slick.dbio.DBIOAction import slick.jdbc.PostgresProfile.api._ @@ -15,6 +16,7 @@ import org.alephium.explorer.GenApiModel._ import org.alephium.explorer.GenDBModel._ import org.alephium.explorer.api.model.Pagination import org.alephium.explorer.persistence.{DatabaseFixtureForEach, TestDBRunner} +import org.alephium.explorer.persistence.model.{FungibleTokenMetadataEntity, NFTMetadataEntity} import org.alephium.explorer.persistence.queries.TokenQueries import org.alephium.explorer.persistence.queries.result.TxByTokenQR import org.alephium.explorer.persistence.schema._ @@ -23,6 +25,23 @@ import org.alephium.util.{TimeStamp, U256} class TokenQueriesSpec extends AlephiumFutureSpec with DatabaseFixtureForEach with TestDBRunner { + private val fungibleMetadataGen = fungibleTokenMetadataGen.map { metadata => + FungibleTokenMetadataEntity( + metadata.id, + ByteString(metadata.symbol), + ByteString(metadata.name), + metadata.decimals + ) + } + private val nftEntityGen = nftMetadataGen.map { metadata => + NFTMetadataEntity( + metadata.id, + ByteString(metadata.tokenUri), + metadata.collectionId, + metadata.nftIndex + ) + } + "Token Queries" should { "list token transactions" in { forAll(Gen.listOfN(30, transactionPerTokenEntityGen()), tokenIdGen) { @@ -114,7 +133,7 @@ class TokenQueriesSpec extends AlephiumFutureSpec with DatabaseFixtureForEach wi } "insert and list fungible token metadata" in { - val tokens = Gen.nonEmptyListOf(fungibleTokenMetadataGen).sample.get + val tokens = Gen.nonEmptyListOf(fungibleMetadataGen).sample.get exec( DBIOAction.sequence(tokens.map(token => TokenQueries.insertFungibleTokenMetadata(token))) @@ -126,7 +145,7 @@ class TokenQueriesSpec extends AlephiumFutureSpec with DatabaseFixtureForEach wi } "insert and list nft metadata" in { - val tokens = Gen.nonEmptyListOf(nftMetadataGen).sample.get + val tokens = Gen.nonEmptyListOf(nftEntityGen).sample.get exec( DBIOAction.sequence(tokens.map(token => TokenQueries.insertNFTMetadata(token))) @@ -138,9 +157,9 @@ class TokenQueriesSpec extends AlephiumFutureSpec with DatabaseFixtureForEach wi } "ignore conflict when inserting fungible token metadata" in { - forAll(fungibleTokenMetadataGen, Gen.alphaNumStr) { case (metadata, symbol) => + forAll(fungibleMetadataGen, Gen.alphaNumStr) { case (metadata, symbol) => exec(TokenQueries.insertFungibleTokenMetadata(metadata)) - exec(TokenQueries.insertFungibleTokenMetadata(metadata.copy(symbol = symbol))) + exec(TokenQueries.insertFungibleTokenMetadata(metadata.copy(symbol = ByteString(symbol)))) val result = exec(TokenQueries.listFungibleTokenMetadataQuery(ArraySeq(metadata.id))) @@ -151,9 +170,9 @@ class TokenQueriesSpec extends AlephiumFutureSpec with DatabaseFixtureForEach wi } "ignore conflict when inserting nft metadata" in { - forAll(nftMetadataGen, Gen.alphaNumStr) { case (metadata, tokenUri) => + forAll(nftEntityGen, Gen.alphaNumStr) { case (metadata, tokenUri) => exec(TokenQueries.insertNFTMetadata(metadata)) - exec(TokenQueries.insertNFTMetadata(metadata.copy(tokenUri = tokenUri))) + exec(TokenQueries.insertNFTMetadata(metadata.copy(tokenUri = ByteString(tokenUri)))) val result = exec(TokenQueries.listNFTMetadataQuery(ArraySeq(metadata.id))) diff --git a/app/src/test/scala/org/alephium/explorer/service/BlockFlowClientSpec.scala b/app/src/test/scala/org/alephium/explorer/service/BlockFlowClientSpec.scala index bc59e098a..33f71b65b 100644 --- a/app/src/test/scala/org/alephium/explorer/service/BlockFlowClientSpec.scala +++ b/app/src/test/scala/org/alephium/explorer/service/BlockFlowClientSpec.scala @@ -8,7 +8,6 @@ import java.net.InetAddress import scala.collection.immutable.ArraySeq import org.apache.pekko.testkit.SocketUtil -import org.apache.pekko.util.ByteString import sttp.model.Uri import org.alephium.api.model @@ -18,7 +17,6 @@ import org.alephium.explorer.GenApiModel._ import org.alephium.explorer.GenCoreApi._ import org.alephium.explorer.GenCoreProtocol._ import org.alephium.explorer.TestBlockFlowServer -import org.alephium.explorer.api.model.{FungibleTokenMetadata, NFTCollectionMetadata, NFTMetadata} import org.alephium.explorer.error.ExplorerError import org.alephium.explorer.persistence.DatabaseFixtureForAll import org.alephium.explorer.persistence.model._ @@ -31,8 +29,6 @@ class BlockFlowClientSpec extends AlephiumFutureSpec with DatabaseFixtureForAll val group = GroupIndex.Zero val localhost: InetAddress = InetAddress.getByName("127.0.0.1") - def bytesToString(bytes: ByteString): String = bytes.utf8String.replaceAll("\u0000", "") - "BlockFlowClient.fetchBlock" should { val port = SocketUtil.temporaryLocalPort(SocketUtil.Both) val block = blockEntryProtocolGen.sample.get @@ -89,10 +85,10 @@ class BlockFlowClientSpec extends AlephiumFutureSpec with DatabaseFixtureForAll val callContract = result.copy(results = results) BlockFlowClient.extractFungibleTokenMetadata(token, callContract) is Some( - FungibleTokenMetadata( + FungibleTokenMetadataEntity( token, - bytesToString(symbol.value), - bytesToString(name.value), + symbol.value, + name.value, decimals.value ) ) @@ -111,9 +107,9 @@ class BlockFlowClientSpec extends AlephiumFutureSpec with DatabaseFixtureForAll val callContract = result.copy(results = results) BlockFlowClient.extractNFTMetadata(token, callContract) is Some( - NFTMetadata( + NFTMetadataEntity( token, - bytesToString(uri.value), + uri.value, ContractId.from(contractId.value).get, nftIndex.value ) @@ -131,9 +127,9 @@ class BlockFlowClientSpec extends AlephiumFutureSpec with DatabaseFixtureForAll val callContract = result.copy(results = results) BlockFlowClient.extractNFTCollectionMetadata(contractAddress, callContract) is Some( - NFTCollectionMetadata( + NFTCollectionMetadataEntity( contractAddress, - bytesToString(uri.value) + uri.value ) ) } diff --git a/app/src/test/scala/org/alephium/explorer/service/EmptyBlockFlowClient.scala b/app/src/test/scala/org/alephium/explorer/service/EmptyBlockFlowClient.scala index 9832558e7..7fd32ae9f 100644 --- a/app/src/test/scala/org/alephium/explorer/service/EmptyBlockFlowClient.scala +++ b/app/src/test/scala/org/alephium/explorer/service/EmptyBlockFlowClient.scala @@ -53,15 +53,17 @@ trait EmptyBlockFlowClient extends BlockFlowClient { override def guessTokenStdInterfaceId(token: TokenId): Future[Option[StdInterfaceId]] = Future.successful(None) - override def fetchFungibleTokenMetadata(token: TokenId): Future[Option[FungibleTokenMetadata]] = + override def fetchFungibleTokenMetadata( + token: TokenId + ): Future[Option[FungibleTokenMetadataEntity]] = Future.successful(None) - override def fetchNFTMetadata(token: TokenId): Future[Option[NFTMetadata]] = + override def fetchNFTMetadata(token: TokenId): Future[Option[NFTMetadataEntity]] = Future.successful(None) override def fetchNFTCollectionMetadata( contract: Address.Contract - ): Future[Option[NFTCollectionMetadata]] = + ): Future[Option[NFTCollectionMetadataEntity]] = Future.successful(None) override def start(): Future[Unit] = ???