Skip to content

Commit e00cf48

Browse files
feat: migrate legacy hf.co tags to huggingface.co during client initialization (#645)
* feat: migrate legacy hf.co tags to huggingface.co during client initialization * Update pkg/distribution/distribution/client_test.go Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * fix format --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1 parent 7a5b9b8 commit e00cf48

4 files changed

Lines changed: 225 additions & 2 deletions

File tree

pkg/distribution/distribution/client.go

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,37 @@ func NewClient(opts ...Option) (*Client, error) {
103103
}
104104

105105
options.logger.Infoln("Successfully initialized store")
106-
return &Client{
106+
c := &Client{
107107
store: s,
108108
log: options.logger,
109109
registry: registryClient,
110-
}, nil
110+
}
111+
112+
// Migrate any legacy hf.co tags to huggingface.co
113+
if err := c.migrateHFTags(); err != nil {
114+
options.logger.Warnf("Failed to migrate HuggingFace tags: %v", err)
115+
}
116+
117+
return c, nil
118+
}
119+
120+
// migrateHFTags normalizes legacy hf.co/ tags in the store to huggingface.co/.
121+
// This handles models that were pulled before the hf.co normalization was added,
122+
// ensuring they can be found by the cache check in PullModel.
123+
func (c *Client) migrateHFTags() error {
124+
migrated, err := c.store.MigrateTags(func(tag string) string {
125+
if rest, found := strings.CutPrefix(tag, "hf.co/"); found {
126+
return "huggingface.co/" + rest
127+
}
128+
return tag
129+
})
130+
if err != nil {
131+
return err
132+
}
133+
if migrated > 0 {
134+
c.log.Infof("Migrated %d HuggingFace tag(s) from hf.co to huggingface.co", migrated)
135+
}
136+
return nil
111137
}
112138

113139
// normalizeModelName adds the default organization prefix (ai/) and tag (:latest) if missing.

pkg/distribution/distribution/client_test.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1141,6 +1141,94 @@ func randomFile(size int64) (string, error) {
11411141
return f.Name(), nil
11421142
}
11431143

1144+
func TestMigrateHFTagsOnClientInit(t *testing.T) {
1145+
testCases := []struct {
1146+
name string
1147+
storedTag string
1148+
lookupRef string
1149+
shouldMigrate bool
1150+
}{
1151+
{
1152+
name: "hf.co tag migrated to huggingface.co on init",
1153+
storedTag: "hf.co/testorg/testmodel:latest",
1154+
lookupRef: "hf.co/testorg/testmodel",
1155+
shouldMigrate: true,
1156+
},
1157+
{
1158+
name: "hf.co tag with quantization migrated",
1159+
storedTag: "hf.co/bartowski/llama-3.2-1b-instruct-gguf:Q4_K_M",
1160+
lookupRef: "hf.co/bartowski/llama-3.2-1b-instruct-gguf:Q4_K_M",
1161+
shouldMigrate: true,
1162+
},
1163+
{
1164+
name: "huggingface.co tag unchanged",
1165+
storedTag: "huggingface.co/testorg/testmodel:latest",
1166+
lookupRef: "huggingface.co/testorg/testmodel",
1167+
shouldMigrate: false,
1168+
},
1169+
}
1170+
1171+
for _, tc := range testCases {
1172+
t.Run(tc.name, func(t *testing.T) {
1173+
tempDir := t.TempDir()
1174+
1175+
// Step 1: Create a client and write a model with the legacy tag
1176+
setupClient, err := newTestClient(tempDir)
1177+
if err != nil {
1178+
t.Fatalf("Failed to create setup client: %v", err)
1179+
}
1180+
1181+
model, err := gguf.NewModel(testGGUFFile)
1182+
if err != nil {
1183+
t.Fatalf("Failed to create model: %v", err)
1184+
}
1185+
1186+
if err := setupClient.store.Write(model, []string{tc.storedTag}, nil); err != nil {
1187+
t.Fatalf("Failed to write model to store: %v", err)
1188+
}
1189+
1190+
// Step 2: Create a NEW client (simulating restart) - migration should happen
1191+
client, err := newTestClient(tempDir)
1192+
if err != nil {
1193+
t.Fatalf("Failed to create client: %v", err)
1194+
}
1195+
1196+
// Step 3: Verify the model can be found using the reference
1197+
// (normalizeModelName converts hf.co -> huggingface.co, and migration should have updated the store)
1198+
foundModel, err := client.GetModel(tc.lookupRef)
1199+
if err != nil {
1200+
t.Fatalf("Failed to get model after migration: %v", err)
1201+
}
1202+
1203+
if foundModel == nil {
1204+
t.Fatal("Expected to find model after migration, got nil")
1205+
}
1206+
1207+
// Step 4: If the tag was hf.co, verify it was actually migrated in the store
1208+
if tc.shouldMigrate {
1209+
// The model should now have huggingface.co tag, not hf.co
1210+
tags := foundModel.Tags()
1211+
hasOldTag := false
1212+
hasNewTag := false
1213+
for _, tag := range tags {
1214+
if strings.HasPrefix(tag, "hf.co/") {
1215+
hasOldTag = true
1216+
}
1217+
if strings.HasPrefix(tag, "huggingface.co/") {
1218+
hasNewTag = true
1219+
}
1220+
}
1221+
if hasOldTag {
1222+
t.Errorf("Model still has old hf.co tag after migration: %v", tags)
1223+
}
1224+
if !hasNewTag {
1225+
t.Errorf("Model doesn't have huggingface.co tag after migration: %v", tags)
1226+
}
1227+
}
1228+
})
1229+
}
1230+
}
1231+
11441232
func TestPullHuggingFaceModelFromCache(t *testing.T) {
11451233
testCases := []struct {
11461234
name string

pkg/distribution/internal/store/store.go

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,39 @@ func (s *LocalStore) WriteLightweight(mdl oci.Image, tags []string) (err error)
534534
return nil
535535
}
536536

537+
// MigrateTags applies a transformation function to all tags in the index.
538+
// If the function returns a different string, the tag is updated.
539+
// Returns the number of tags that were migrated.
540+
func (s *LocalStore) MigrateTags(transform func(string) string) (int, error) {
541+
index, err := s.readIndex()
542+
if err != nil {
543+
return 0, fmt.Errorf("reading index for migration: %w", err)
544+
}
545+
546+
migrated := 0
547+
changed := false
548+
for i, entry := range index.Models {
549+
newTags := make([]string, len(entry.Tags))
550+
for j, tag := range entry.Tags {
551+
newTag := transform(tag)
552+
if newTag != tag {
553+
migrated++
554+
changed = true
555+
}
556+
newTags[j] = newTag
557+
}
558+
index.Models[i].Tags = newTags
559+
}
560+
561+
if changed {
562+
if err := s.writeIndex(index); err != nil {
563+
return 0, fmt.Errorf("writing migrated index: %w", err)
564+
}
565+
}
566+
567+
return migrated, nil
568+
}
569+
537570
// Read reads a model from the store by reference (either tag or ID)
538571
func (s *LocalStore) Read(reference string) (*Model, error) {
539572
models, err := s.List()

pkg/distribution/internal/store/store_test.go

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -991,6 +991,82 @@ func TestResetStore(t *testing.T) {
991991
}
992992
}
993993

994+
func TestMigrateTags(t *testing.T) {
995+
tempDir := t.TempDir()
996+
997+
storePath := filepath.Join(tempDir, "migrate-tags-store")
998+
s, err := store.New(store.Options{
999+
RootPath: storePath,
1000+
})
1001+
if err != nil {
1002+
t.Fatalf("Failed to create store: %v", err)
1003+
}
1004+
1005+
// Write a model with an hf.co tag (simulating pre-migration state)
1006+
mdl := newTestModel(t)
1007+
if err := s.Write(mdl, []string{"hf.co/testorg/testmodel:latest"}, nil); err != nil {
1008+
t.Fatalf("Write failed: %v", err)
1009+
}
1010+
1011+
// Write another model with a non-HF tag (should be unaffected)
1012+
mdl2Content := []byte("another model content")
1013+
mdl2Path := filepath.Join(tempDir, "other-model.gguf")
1014+
if err := os.WriteFile(mdl2Path, mdl2Content, 0644); err != nil {
1015+
t.Fatalf("Failed to write model file: %v", err)
1016+
}
1017+
mdl2, err := gguf.NewModel(mdl2Path)
1018+
if err != nil {
1019+
t.Fatalf("Failed to create model: %v", err)
1020+
}
1021+
if err := s.Write(mdl2, []string{"ai/some-model:latest"}, nil); err != nil {
1022+
t.Fatalf("Write failed: %v", err)
1023+
}
1024+
1025+
// Run migration: hf.co/ -> huggingface.co/
1026+
migrated, err := s.MigrateTags(func(tag string) string {
1027+
if rest, found := strings.CutPrefix(tag, "hf.co/"); found {
1028+
return "huggingface.co/" + rest
1029+
}
1030+
return tag
1031+
})
1032+
if err != nil {
1033+
t.Fatalf("MigrateTags failed: %v", err)
1034+
}
1035+
1036+
if migrated != 1 {
1037+
t.Errorf("Expected 1 migrated tag, got %d", migrated)
1038+
}
1039+
1040+
// Verify the model can be found with the new tag
1041+
if _, err := s.Read("huggingface.co/testorg/testmodel:latest"); err != nil {
1042+
t.Fatalf("Failed to read model with migrated tag: %v", err)
1043+
}
1044+
1045+
// Verify the old tag no longer works
1046+
if _, err := s.Read("hf.co/testorg/testmodel:latest"); !errors.Is(err, store.ErrModelNotFound) {
1047+
t.Errorf("Expected ErrModelNotFound for old tag, got: %v", err)
1048+
}
1049+
1050+
// Verify the non-HF model is unaffected
1051+
if _, err := s.Read("ai/some-model:latest"); err != nil {
1052+
t.Fatalf("Non-HF model should be unaffected: %v", err)
1053+
}
1054+
1055+
// Running migration again should be a no-op
1056+
migrated2, err := s.MigrateTags(func(tag string) string {
1057+
if rest, found := strings.CutPrefix(tag, "hf.co/"); found {
1058+
return "huggingface.co/" + rest
1059+
}
1060+
return tag
1061+
})
1062+
if err != nil {
1063+
t.Fatalf("Second MigrateTags failed: %v", err)
1064+
}
1065+
if migrated2 != 0 {
1066+
t.Errorf("Expected 0 migrated tags on second run, got %d", migrated2)
1067+
}
1068+
}
1069+
9941070
func TestWriteLightweight(t *testing.T) {
9951071
tempDir := t.TempDir()
9961072

0 commit comments

Comments
 (0)