Skip to content

feat: added verify command to check if URLs in metadata are reachable #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
quintesse merged 1 commit into from
Aug 26, 2026
Merged

feat: added verify command to check if URLs in metadata are reachable #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file added .bob/notes/pending-notes.txt
View file
Open in desktop
Empty file.
56 changes: 8 additions & 48 deletions src/main/java/dev/jbang/jdkdb/DownloadCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,10 @@
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -112,7 +110,8 @@ public Integer call() throws Exception {
GitHubUtils.setupGitHubToken();

// Process file type filter
Set<JdkMetadata.FileType> fileTypeFilter = processFileTypeFilter(includeFileTypes, excludeFileTypes);
Set<JdkMetadata.FileType> fileTypeFilter =
JdkMetadata.processFileTypeFilter(includeFileTypes, excludeFileTypes);

logger.info("Java Metadata Scraper - Download");
logger.info("=================================");
Expand Down Expand Up @@ -295,37 +294,6 @@ static List<JdkMetadata> prioritizeMetadata(List<JdkMetadata> metadataList, bool
return ordered;
}

/**
* Process the include and exclude file type options to create a filter set.
*
* @param includeFileTypes List of file types to include (null or empty means include all)
* @param excludeFileTypes List of file types to exclude (null or empty means exclude none)
* @return A set of file types to accept, or null if no filtering should be applied
*/
private Set<JdkMetadata.FileType> processFileTypeFilter(
List<JdkMetadata.FileType> includeFileTypes, List<JdkMetadata.FileType> excludeFileTypes) {
if ((includeFileTypes == null || includeFileTypes.isEmpty())
&& (excludeFileTypes == null || excludeFileTypes.isEmpty())) {
return null; // No filtering
}

Set<JdkMetadata.FileType> result;
if (includeFileTypes != null && !includeFileTypes.isEmpty()) {
// Start with only the included types
result = EnumSet.copyOf(includeFileTypes);
} else {
// Start with all types
result = EnumSet.allOf(JdkMetadata.FileType.class);
}

// Remove excluded types
if (excludeFileTypes != null && !excludeFileTypes.isEmpty()) {
result.removeAll(excludeFileTypes);
}

return result.isEmpty() ? null : result;
}

public static class DownloadProcessor implements DefaultDownloadManager.DownloadProcessor {
private final HttpUtils httpUtils;
private final Path metadataDir;
Expand All @@ -344,7 +312,7 @@ public void processDownload(DownloadTask task) throws IOException, InterruptedEx
JdkMetadata metadata = task.metadata();
String filename = metadata.getFilename();
String url = metadata.getUrl();
Optional<String> unlistedSince = findUnlistedSince(metadata);
String today = java.time.LocalDate.now().toString();

if (filename == null || url == null) {
return;
Expand All @@ -358,28 +326,27 @@ public void processDownload(DownloadTask task) throws IOException, InterruptedEx
Path tempFile = Files.createTempFile("jdk-metadata-", "-" + filename);

try {
if (unlistedSince.isPresent()) {
if (metadata.getUnlistedSince() != null) {
task.downloadLogger()
.info(
"Metadata item {} has unlisted_since={} - attempting download",
filename,
unlistedSince.get());
metadata.getUnlistedSince());
}
task.downloadLogger().info("Downloading " + filename);
try {
httpUtils.downloadFile(url, tempFile);
} catch (IOException e) {
if (isHttpStatus(e, 403, 404)) {
if (unlistedSince.isPresent()) {
if (metadata.getUnlistedSince() != null) {
task.downloadLogger()
.warn(
"Download returned 40X for unlisted package {} (unlisted_since={}). "
+ "This package is most likely not available anymore and is a candidate for pruning.",
filename,
unlistedSince.get());
metadata.getUnlistedSince());
}
if (markMissing && metadata.getMissingSince() == null) {
String today = java.time.LocalDate.now().toString();
metadata.setMissingSince(today);
task.downloadLogger().warn("Marking {} as missing_since={}", filename, today);
saveMetadata(task, metadataDir, metadata);
Expand Down Expand Up @@ -412,6 +379,7 @@ public void processDownload(DownloadTask task) throws IOException, InterruptedEx
task.downloadLogger().info("Clearing missing_since for {}", filename);
metadata.setMissingSince(null);
}
metadata.setLastVerified(today);

// Extract and parse release info from archive
try {
Expand Down Expand Up @@ -464,14 +432,6 @@ private static void saveChecksumFile(Path checksumDir, String filename, String a
Files.writeString(checksumFile, checksum + " " + filename + "\n");
}

private static Optional<String> findUnlistedSince(JdkMetadata metadata) {
String value = metadata.getUnlistedSince();
if (value == null || value.isBlank()) {
return Optional.empty();
}
return Optional.of(value);
}

private static boolean isHttpStatus(IOException e, int... statusCodes) {
if (!(e instanceof HttpStatusException hse)) {
return false;
Expand Down
1 change: 1 addition & 0 deletions src/main/java/dev/jbang/jdkdb/MainCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
UpdateCommand.class,
IndexCommand.class,
DownloadCommand.class,
VerifyCommand.class,
CleanCommand.class,
ConvertCommand.class
})
Expand Down
Loading