Preserve raw token and NFT metadata in PostgreSQL bytea

README.md

@@ -204,3 +204,19 @@ apiMappings ++=
     )
   )
 ```
+
+### Token metadata storage migration
+
+Database migration 12 stores fungible token symbols and names, NFT token URIs,
+and NFT collection URIs as PostgreSQL `bytea`. Startup converts existing text
+columns using UTF-8 encoding; literal backslashes remain literal data. Columns
+already stored as `bytea` are left unchanged, including on migration replay.
+The migration runs in the normal migration transaction and may lock and rewrite
+the affected tables. Schedule upgrades with their size in mind.
+
+New metadata is persisted exactly as returned by contract calls, including NUL
+bytes and malformed UTF-8. The API still returns the same string fields: UTF-8
+decoding happens when producing API models, with replacement characters for
+malformed sequences. NUL characters are retained and escaped by JSON encoding.
+The earlier metadata conversion stripped NUL characters; migration cannot recover
+bytes already removed or replaced in existing text records.

app/src/main/scala/org/alephium/explorer/persistence/Migrations.scala

@@ -29,7 +29,7 @@ import org.alephium.util.discard
 object Migrations extends StrictLogging {
 
   // scalastyle:off magic.number
-  val latestVersion: MigrationVersion = MigrationVersion(11)
+  val latestVersion: MigrationVersion = MigrationVersion(12)
 
   def migration1(implicit ec: ExecutionContext): DBActionAll[Unit] = {
     // We retrigger the download of fungible and non-fungible tokens' metadata that have sub-category
@@ -222,6 +222,34 @@ object Migrations extends StrictLogging {
     """.map(_ => ())
   }
 
+  // Existing text is encoded as UTF-8, not interpreted as PostgreSQL bytea input syntax.
+  // The migration can be replayed after a reset migration counter and on fresh bytea schemas.
+  def migration12(implicit ec: ExecutionContext): DBActionAll[Unit] = {
+    DBIO.seq(
+      metadataColumnToBytes("fungible_token_metadata", "symbol"),
+      metadataColumnToBytes("fungible_token_metadata", "name"),
+      metadataColumnToBytes("nft_metadata", "token_uri"),
+      metadataColumnToBytes("nft_collection_metadata", "collection_uri")
+    )
+  }
+
+  private def metadataColumnToBytes(table: String, column: String)(implicit
+      ec: ExecutionContext
+  ): DBActionAll[Unit] = {
+    sql"""SELECT data_type FROM information_schema.columns
+          WHERE table_schema = current_schema() AND table_name = $table AND column_name = $column"""
+      .as[String]
+      .head
+      .flatMap {
+        case "bytea" => DBIO.successful(())
+        case "text" | "character varying" | "character" =>
+          sqlu"""ALTER TABLE #$table ALTER COLUMN #$column TYPE bytea
+                 USING convert_to(#$column, 'UTF8')""".map(_ => ())
+        case other =>
+          DBIO.failed(new IllegalStateException(s"Unexpected metadata column type: $other"))
+      }
+  }
+
   private def migrations(implicit
       explorerConfig: ExplorerConfig,
       ec: ExecutionContext
@@ -236,7 +264,8 @@ object Migrations extends StrictLogging {
     migration8,
     migration9,
     migration10,
-    migration11
+    migration11,
+    migration12
   )
 
   def backgroundCoinbaseMigration()(implicit

app/src/main/scala/org/alephium/explorer/persistence/model/FungibleTokenMetadataEntity.scala

@@ -0,0 +1,21 @@
+// Copyright (c) Alephium
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package org.alephium.explorer.persistence.model
+
+import org.apache.pekko.util.ByteString
+
+import org.alephium.explorer.api.model.FungibleTokenMetadata
+import org.alephium.protocol.model.TokenId
+import org.alephium.util.U256
+
+// Preserve arbitrary contract bytes until constructing the public API representation.
+final case class FungibleTokenMetadataEntity(
+    id: TokenId,
+    symbol: ByteString,
+    name: ByteString,
+    decimals: U256
+) {
+  def toApi(): FungibleTokenMetadata =
+    FungibleTokenMetadata(id, symbol.utf8String, name.utf8String, decimals)
+}

app/src/main/scala/org/alephium/explorer/persistence/model/NFTCollectionMetadataEntity.scala

@@ -0,0 +1,17 @@
+// Copyright (c) Alephium
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package org.alephium.explorer.persistence.model
+
+import org.apache.pekko.util.ByteString
+
+import org.alephium.explorer.api.model.NFTCollectionMetadata
+import org.alephium.protocol.model.Address
+
+// Preserve arbitrary contract bytes until constructing the public API representation.
+final case class NFTCollectionMetadataEntity(
+    address: Address.Contract,
+    collectionUri: ByteString
+) {
+  def toApi(): NFTCollectionMetadata = NFTCollectionMetadata(address, collectionUri.utf8String)
+}

app/src/main/scala/org/alephium/explorer/persistence/model/NFTMetadataEntity.scala

@@ -0,0 +1,20 @@
+// Copyright (c) Alephium
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package org.alephium.explorer.persistence.model
+
+import org.apache.pekko.util.ByteString
+
+import org.alephium.explorer.api.model.NFTMetadata
+import org.alephium.protocol.model.{ContractId, TokenId}
+import org.alephium.util.U256
+
+// Preserve arbitrary contract bytes until constructing the public API representation.
+final case class NFTMetadataEntity(
+    id: TokenId,
+    tokenUri: ByteString,
+    collectionId: ContractId,
+    nftIndex: U256
+) {
+  def toApi(): NFTMetadata = NFTMetadata(id, tokenUri.utf8String, collectionId, nftIndex)
+}

app/src/main/scala/org/alephium/explorer/persistence/queries/ContractQueries.scala

@@ -11,7 +11,11 @@ import slick.jdbc.PostgresProfile.api._
 
 import org.alephium.explorer.api.model._
 import org.alephium.explorer.persistence._
-import org.alephium.explorer.persistence.model.{ContractEntity, EventEntity}
+import org.alephium.explorer.persistence.model.{
+  ContractEntity,
+  EventEntity,
+  NFTCollectionMetadataEntity
+}
 import org.alephium.explorer.persistence.schema.CustomGetResult._
 import org.alephium.explorer.persistence.schema.CustomSetParameter._
 import org.alephium.explorer.util.SlickUtil._
@@ -160,7 +164,7 @@ object ContractQueries {
 
   def insertNFTCollectionMetadata(
       contract: Address,
-      metadata: NFTCollectionMetadata
+      metadata: NFTCollectionMetadataEntity
   ): DBActionW[Int] = {
     sqlu"""
       INSERT INTO nft_collection_metadata (

app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala

@@ -14,7 +14,12 @@ import slick.jdbc.PostgresProfile.api._
 import org.alephium.api.model.{Address => ApiAddress}
 import org.alephium.explorer.api.model._
 import org.alephium.explorer.persistence._
-import org.alephium.explorer.persistence.model.TokenInfoEntity
+import org.alephium.explorer.persistence.model.{
+  FungibleTokenMetadataEntity,
+  NFTCollectionMetadataEntity,
+  NFTMetadataEntity,
+  TokenInfoEntity
+}
 import org.alephium.explorer.persistence.queries.result.TxByTokenQR
 import org.alephium.explorer.persistence.schema.CustomGetResult._
 import org.alephium.explorer.persistence.schema.CustomSetParameter._
@@ -221,7 +226,7 @@ object TokenQueries extends StrictLogging {
       .asAS[(TokenId, Option[U256], Option[U256])]
 
   def insertFungibleTokenMetadata(
-      metadata: FungibleTokenMetadata
+      metadata: FungibleTokenMetadataEntity
   ): DBActionW[Int] = {
     sqlu"""
       INSERT INTO fungible_token_metadata (
@@ -235,7 +240,7 @@ object TokenQueries extends StrictLogging {
 
   def listFungibleTokenMetadataQuery(
       tokens: ArraySeq[TokenId]
-  ): DBActionW[ArraySeq[FungibleTokenMetadata]] = {
+  ): DBActionW[ArraySeq[FungibleTokenMetadataEntity]] = {
     if (tokens.nonEmpty) {
       val params = paramPlaceholder(1, tokens.size)
 
@@ -255,7 +260,7 @@ object TokenQueries extends StrictLogging {
       SQLActionBuilder(
         sql = query,
         setParameter = parameters
-      ).asASE[FungibleTokenMetadata](fungibleTokenMetadataGetResult)
+      ).asASE[FungibleTokenMetadataEntity](fungibleTokenMetadataGetResult)
     } else {
       DBIOAction.successful(ArraySeq.empty)
     }
@@ -291,7 +296,7 @@ object TokenQueries extends StrictLogging {
 
   def listNFTMetadataQuery(
       tokens: ArraySeq[TokenId]
-  ): DBActionW[ArraySeq[NFTMetadata]] = {
+  ): DBActionW[ArraySeq[NFTMetadataEntity]] = {
     if (tokens.nonEmpty) {
       val params = paramPlaceholder(1, tokens.size)
 
@@ -311,15 +316,15 @@ object TokenQueries extends StrictLogging {
       SQLActionBuilder(
         sql = query,
         setParameter = parameters
-      ).asASE[NFTMetadata](nftMetadataGetResult)
+      ).asASE[NFTMetadataEntity](nftMetadataGetResult)
     } else {
       DBIOAction.successful(ArraySeq.empty)
     }
   }
 
   def listNFTCollectionMetadataQuery(
       addresses: ArraySeq[Address.Contract]
-  ): DBActionW[ArraySeq[NFTCollectionMetadata]] = {
+  ): DBActionW[ArraySeq[NFTCollectionMetadataEntity]] = {
     if (addresses.nonEmpty) {
       val params = paramPlaceholder(1, addresses.size)
 
@@ -339,14 +344,14 @@ object TokenQueries extends StrictLogging {
       SQLActionBuilder(
         sql = query,
         setParameter = parameters
-      ).asASE[NFTCollectionMetadata](nftCollectionMetadataGetResult)
+      ).asASE[NFTCollectionMetadataEntity](nftCollectionMetadataGetResult)
     } else {
       DBIOAction.successful(ArraySeq.empty)
     }
   }
 
   def insertNFTMetadata(
-      metadata: NFTMetadata
+      metadata: NFTMetadataEntity
   ): DBActionW[Int] = {
     sqlu"""
       INSERT INTO nft_metadata (

app/src/main/scala/org/alephium/explorer/persistence/schema/CustomGetResult.scala

@@ -398,27 +398,27 @@ object CustomGetResult {
         eventOrder = result.<<
       )
 
-  val fungibleTokenMetadataGetResult: GetResult[FungibleTokenMetadata] =
+  val fungibleTokenMetadataGetResult: GetResult[FungibleTokenMetadataEntity] =
     (result: PositionedResult) =>
-      FungibleTokenMetadata(
+      FungibleTokenMetadataEntity(
         id = result.<<,
         symbol = result.<<,
         name = result.<<,
         decimals = result.<<
       )
 
-  val nftMetadataGetResult: GetResult[NFTMetadata] =
+  val nftMetadataGetResult: GetResult[NFTMetadataEntity] =
     (result: PositionedResult) =>
-      NFTMetadata(
+      NFTMetadataEntity(
         id = result.<<,
         tokenUri = result.<<,
         collectionId = result.<<,
         nftIndex = result.<<
       )
 
-  val nftCollectionMetadataGetResult: GetResult[NFTCollectionMetadata] =
+  val nftCollectionMetadataGetResult: GetResult[NFTCollectionMetadataEntity] =
     (result: PositionedResult) =>
-      NFTCollectionMetadata(
+      NFTCollectionMetadataEntity(
         address = result.<<(addressContractGetResult),
         collectionUri = result.<<
       )

app/src/main/scala/org/alephium/explorer/persistence/schema/FungibleTokenMetadataSchema.scala

@@ -3,26 +3,27 @@
 
 package org.alephium.explorer.persistence.schema
 
+import org.apache.pekko.util.ByteString
 import slick.jdbc.PostgresProfile.api._
 import slick.lifted.ProvenShape
 
-import org.alephium.explorer.api.model.FungibleTokenMetadata
+import org.alephium.explorer.persistence.model.FungibleTokenMetadataEntity
 import org.alephium.explorer.persistence.schema.CustomJdbcTypes._
 import org.alephium.protocol.model.TokenId
 import org.alephium.util.U256
 
 object FungibleTokenMetadataSchema
-    extends SchemaMainChain[FungibleTokenMetadata]("fungible_token_metadata") {
+    extends SchemaMainChain[FungibleTokenMetadataEntity]("fungible_token_metadata") {
 
-  class TokenInfos(tag: Tag) extends Table[FungibleTokenMetadata](tag, name) {
-    def token: Rep[TokenId] = column[TokenId]("token", O.PrimaryKey)
-    def symbol: Rep[String] = column[String]("symbol")
-    def name: Rep[String]   = column[String]("name")
-    def decimals: Rep[U256] = column[U256]("decimals", O.SqlType("DECIMAL(80,0)"))
+  class TokenInfos(tag: Tag) extends Table[FungibleTokenMetadataEntity](tag, name) {
+    def token: Rep[TokenId]     = column[TokenId]("token", O.PrimaryKey)
+    def symbol: Rep[ByteString] = column[ByteString]("symbol")
+    def name: Rep[ByteString]   = column[ByteString]("name")
+    def decimals: Rep[U256]     = column[U256]("decimals", O.SqlType("DECIMAL(80,0)"))
 
-    def * : ProvenShape[FungibleTokenMetadata] =
+    def * : ProvenShape[FungibleTokenMetadataEntity] =
       (token, symbol, name, decimals)
-        .<>((FungibleTokenMetadata.apply _).tupled, FungibleTokenMetadata.unapply)
+        .<>((FungibleTokenMetadataEntity.apply _).tupled, FungibleTokenMetadataEntity.unapply)
   }
 
   val table: TableQuery[TokenInfos] = TableQuery[TokenInfos]

app/src/main/scala/org/alephium/explorer/persistence/schema/NFTCollectionMetadataSchema.scala

@@ -3,23 +3,24 @@
 
 package org.alephium.explorer.persistence.schema
 
+import org.apache.pekko.util.ByteString
 import slick.jdbc.PostgresProfile.api._
 import slick.lifted.ProvenShape
 
-import org.alephium.explorer.api.model.NFTCollectionMetadata
+import org.alephium.explorer.persistence.model.NFTCollectionMetadataEntity
 import org.alephium.explorer.persistence.schema.CustomJdbcTypes._
 import org.alephium.protocol.model.Address
 
 object NFTCollectionMetadataSchema
-    extends SchemaMainChain[NFTCollectionMetadata]("nft_collection_metadata") {
+    extends SchemaMainChain[NFTCollectionMetadataEntity]("nft_collection_metadata") {
 
-  class TokenInfos(tag: Tag) extends Table[NFTCollectionMetadata](tag, name) {
+  class TokenInfos(tag: Tag) extends Table[NFTCollectionMetadataEntity](tag, name) {
     def contract: Rep[Address.Contract] = column[Address.Contract]("contract", O.PrimaryKey)
-    def collectionUri: Rep[String]      = column[String]("collection_uri")
+    def collectionUri: Rep[ByteString]  = column[ByteString]("collection_uri")
 
-    def * : ProvenShape[NFTCollectionMetadata] =
+    def * : ProvenShape[NFTCollectionMetadataEntity] =
       (contract, collectionUri)
-        .<>((NFTCollectionMetadata.apply _).tupled, NFTCollectionMetadata.unapply)
+        .<>((NFTCollectionMetadataEntity.apply _).tupled, NFTCollectionMetadataEntity.unapply)
   }
 
   val table: TableQuery[TokenInfos] = TableQuery[TokenInfos]

app/src/main/scala/org/alephium/explorer/persistence/schema/NFTMetadataSchema.scala

@@ -3,25 +3,26 @@
 
 package org.alephium.explorer.persistence.schema
 
+import org.apache.pekko.util.ByteString
 import slick.jdbc.PostgresProfile.api._
 import slick.lifted.ProvenShape
 
-import org.alephium.explorer.api.model.NFTMetadata
+import org.alephium.explorer.persistence.model.NFTMetadataEntity
 import org.alephium.explorer.persistence.schema.CustomJdbcTypes._
 import org.alephium.protocol.model.{ContractId, TokenId}
 import org.alephium.util.U256
 
-object NFTMetadataSchema extends SchemaMainChain[NFTMetadata]("nft_metadata") {
+object NFTMetadataSchema extends SchemaMainChain[NFTMetadataEntity]("nft_metadata") {
 
-  class NFTMetadatas(tag: Tag) extends Table[NFTMetadata](tag, name) {
+  class NFTMetadatas(tag: Tag) extends Table[NFTMetadataEntity](tag, name) {
     def token: Rep[TokenId]           = column[TokenId]("token", O.PrimaryKey)
-    def tokenUri: Rep[String]         = column[String]("token_uri")
+    def tokenUri: Rep[ByteString]     = column[ByteString]("token_uri")
     def collectionId: Rep[ContractId] = column[ContractId]("collection_id")
     def nftIndex: Rep[U256]           = column[U256]("nft_index", O.SqlType("DECIMAL(80,0)"))
 
-    def * : ProvenShape[NFTMetadata] =
+    def * : ProvenShape[NFTMetadataEntity] =
       (token, tokenUri, collectionId, nftIndex)
-        .<>((NFTMetadata.apply _).tupled, NFTMetadata.unapply)
+        .<>((NFTMetadataEntity.apply _).tupled, NFTMetadataEntity.unapply)
   }
 
   val table: TableQuery[NFTMetadatas] = TableQuery[NFTMetadatas]