lan/HMCL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature: 下载并发控制 (#5026)

Browse Source
This commit is contained in:
Burning_TNT
2026-01-11 21:45:31 +08:00
committed by GitHub
parent 74d0a89820
commit 3860e68caa
5 changed files with 181 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
HMCL/src/main/java/org/jackhuang/hmcl/setting/ProxyManager.java
View File

@@ -18,6 +18,7 @@
package org.jackhuang.hmcl.setting;
import javafx.beans.InvalidationListener;
import org.jackhuang.hmcl.task.FetchTask;
import org.jackhuang.hmcl.util.StringUtils;
import org.jackhuang.hmcl.util.io.NetworkUtils;
import org.jetbrains.annotations.NotNull;
@@ -114,6 +115,8 @@ public final class ProxyManager {
config().hasProxyAuthProperty().addListener(updateAuthenticator);
config().proxyUserProperty().addListener(updateAuthenticator);
config().proxyPassProperty().addListener(updateAuthenticator);
FetchTask.notifyInitialized();
}
private static abstract class AbstractProxySelector extends ProxySelector {

3
HMCL/src/main/java/org/jackhuang/hmcl/ui/FXUtils.java
View File

@@ -1217,12 +1217,14 @@ public final class FXUtils {
public static Task<Image> getRemoteImageTask(String url, int requestedWidth, int requestedHeight, boolean preserveRatio, boolean smooth) {
return new CacheFileTask(url)
.setSignificance(Task.TaskSignificance.MINOR)
.thenApplyAsync(file -> loadImage(file, requestedWidth, requestedHeight, preserveRatio, smooth))
.setSignificance(Task.TaskSignificance.MINOR);
}
public static Task<Image> getRemoteImageTask(URI uri, int requestedWidth, int requestedHeight, boolean preserveRatio, boolean smooth) {
return new CacheFileTask(uri)
.setSignificance(Task.TaskSignificance.MINOR)
.thenApplyAsync(file -> loadImage(file, requestedWidth, requestedHeight, preserveRatio, smooth))
.setSignificance(Task.TaskSignificance.MINOR);
}
@@ -1237,6 +1239,7 @@ public final class FXUtils {
LOG.warning("An exception encountered while loading remote image: " + url, exception);
}
})
.setSignificance(Task.TaskSignificance.MINOR)
.start();
return image;
}

161
HMCLCore/src/main/java/org/jackhuang/hmcl/mod/curse/CurseForgeRemoteModRepository.java
View File

@@ -35,7 +35,15 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Semaphore;
import java.util.stream.Stream;
import static org.jackhuang.hmcl.util.Lang.mapOf;
@@ -46,6 +54,7 @@ public final class CurseForgeRemoteModRepository implements RemoteModRepository
private static final String PREFIX = "https://api.curseforge.com";
private static final String apiKey = System.getProperty("hmcl.curseforge.apikey", JarUtils.getAttribute("hmcl.curseforge.apikey", ""));
private static final Semaphore SEMAPHORE = new Semaphore(16);
private static final int WORD_PERFECT_MATCH_WEIGHT = 5;
@@ -110,46 +119,51 @@ public final class CurseForgeRemoteModRepository implements RemoteModRepository
@Override
public SearchResult search(DownloadProvider downloadProvider, String gameVersion, @Nullable RemoteModRepository.Category category, int pageOffset, int pageSize, String searchFilter, SortType sortType, SortOrder sortOrder) throws IOException {
int categoryId = 0;
if (category != null && category.getSelf() instanceof CurseAddon.Category) {
categoryId = ((CurseAddon.Category) category.getSelf()).getId();
}
Response<List<CurseAddon>> response = withApiKey(HttpRequest.GET(downloadProvider.injectURL(NetworkUtils.withQuery(PREFIX + "/v1/mods/search", mapOf(
pair("gameId", "432"),
pair("classId", Integer.toString(section)),
pair("categoryId", Integer.toString(categoryId)),
pair("gameVersion", gameVersion),
pair("searchFilter", searchFilter),
pair("sortField", Integer.toString(toModsSearchSortField(sortType))),
pair("sortOrder", toSortOrder(sortOrder)),
pair("index", Integer.toString(pageOffset * pageSize)),
pair("pageSize", Integer.toString(pageSize)))))))
.getJson(Response.typeOf(listTypeOf(CurseAddon.class)));
if (searchFilter.isEmpty()) {
return new SearchResult(response.getData().stream().map(CurseAddon::toMod), calculateTotalPages(response, pageSize));
}
// https://github.com/HMCL-dev/HMCL/issues/1549
String lowerCaseSearchFilter = searchFilter.toLowerCase(Locale.ROOT);
Map<String, Integer> searchFilterWords = new HashMap<>();
for (String s : StringUtils.tokenize(lowerCaseSearchFilter)) {
searchFilterWords.put(s, searchFilterWords.getOrDefault(s, 0) + 1);
}
StringUtils.LevCalculator levCalculator = new StringUtils.LevCalculator();
return new SearchResult(response.getData().stream().map(CurseAddon::toMod).map(remoteMod -> {
String lowerCaseResult = remoteMod.getTitle().toLowerCase(Locale.ROOT);
int diff = levCalculator.calc(lowerCaseSearchFilter, lowerCaseResult);
for (String s : StringUtils.tokenize(lowerCaseResult)) {
if (searchFilterWords.containsKey(s)) {
diff -= WORD_PERFECT_MATCH_WEIGHT * searchFilterWords.get(s) * s.length();
}
SEMAPHORE.acquireUninterruptibly();
try {
int categoryId = 0;
if (category != null && category.getSelf() instanceof CurseAddon.Category) {
categoryId = ((CurseAddon.Category) category.getSelf()).getId();
}
Response<List<CurseAddon>> response = withApiKey(HttpRequest.GET(downloadProvider.injectURL(NetworkUtils.withQuery(PREFIX + "/v1/mods/search", mapOf(
pair("gameId", "432"),
pair("classId", Integer.toString(section)),
pair("categoryId", Integer.toString(categoryId)),
pair("gameVersion", gameVersion),
pair("searchFilter", searchFilter),
pair("sortField", Integer.toString(toModsSearchSortField(sortType))),
pair("sortOrder", toSortOrder(sortOrder)),
pair("index", Integer.toString(pageOffset * pageSize)),
pair("pageSize", Integer.toString(pageSize)))))))
.getJson(Response.typeOf(listTypeOf(CurseAddon.class)));
if (searchFilter.isEmpty()) {
return new SearchResult(response.getData().stream().map(CurseAddon::toMod), calculateTotalPages(response, pageSize));
}
return pair(remoteMod, diff);
}).sorted(Comparator.comparingInt(Pair::getValue)).map(Pair::getKey), response.getData().stream().map(CurseAddon::toMod), calculateTotalPages(response, pageSize));
// https://github.com/HMCL-dev/HMCL/issues/1549
String lowerCaseSearchFilter = searchFilter.toLowerCase(Locale.ROOT);
Map<String, Integer> searchFilterWords = new HashMap<>();
for (String s : StringUtils.tokenize(lowerCaseSearchFilter)) {
searchFilterWords.put(s, searchFilterWords.getOrDefault(s, 0) + 1);
}
StringUtils.LevCalculator levCalculator = new StringUtils.LevCalculator();
return new SearchResult(response.getData().stream().map(CurseAddon::toMod).map(remoteMod -> {
String lowerCaseResult = remoteMod.getTitle().toLowerCase(Locale.ROOT);
int diff = levCalculator.calc(lowerCaseSearchFilter, lowerCaseResult);
for (String s : StringUtils.tokenize(lowerCaseResult)) {
if (searchFilterWords.containsKey(s)) {
diff -= WORD_PERFECT_MATCH_WEIGHT * searchFilterWords.get(s) * s.length();
}
}
return pair(remoteMod, diff);
}).sorted(Comparator.comparingInt(Pair::getValue)).map(Pair::getKey), response.getData().stream().map(CurseAddon::toMod), calculateTotalPages(response, pageSize));
} finally {
SEMAPHORE.release();
}
}
@Override
@@ -173,48 +187,69 @@ public final class CurseForgeRemoteModRepository implements RemoteModRepository
return Optional.empty();
}
Response<FingerprintMatchesResult> response = withApiKey(HttpRequest.POST(PREFIX + "/v1/fingerprints/432"))
.json(mapOf(pair("fingerprints", Collections.singletonList(hash))))
.getJson(Response.typeOf(FingerprintMatchesResult.class));
SEMAPHORE.acquireUninterruptibly();
try {
Response<FingerprintMatchesResult> response = withApiKey(HttpRequest.POST(PREFIX + "/v1/fingerprints/432"))
.json(mapOf(pair("fingerprints", Collections.singletonList(hash))))
.getJson(Response.typeOf(FingerprintMatchesResult.class));
if (response.getData().getExactMatches() == null || response.getData().getExactMatches().isEmpty()) {
return Optional.empty();
if (response.getData().getExactMatches() == null || response.getData().getExactMatches().isEmpty()) {
return Optional.empty();
}
return Optional.of(response.getData().getExactMatches().get(0).getFile().toVersion());
} finally {
SEMAPHORE.release();
}
return Optional.of(response.getData().getExactMatches().get(0).getFile().toVersion());
}
@Override
public RemoteMod getModById(String id) throws IOException {
Response<CurseAddon> response = withApiKey(HttpRequest.GET(PREFIX + "/v1/mods/" + id))
.getJson(Response.typeOf(CurseAddon.class));
return response.data.toMod();
SEMAPHORE.acquireUninterruptibly();
try {
Response<CurseAddon> response = withApiKey(HttpRequest.GET(PREFIX + "/v1/mods/" + id))
.getJson(Response.typeOf(CurseAddon.class));
return response.data.toMod();
} finally {
SEMAPHORE.release();
}
}
@Override
public RemoteMod.File getModFile(String modId, String fileId) throws IOException {
Response<CurseAddon.LatestFile> response = withApiKey(HttpRequest.GET(String.format("%s/v1/mods/%s/files/%s", PREFIX, modId, fileId)))
.getJson(Response.typeOf(CurseAddon.LatestFile.class));
return response.getData().toVersion().getFile();
SEMAPHORE.acquireUninterruptibly();
try {
Response<CurseAddon.LatestFile> response = withApiKey(HttpRequest.GET(String.format("%s/v1/mods/%s/files/%s", PREFIX, modId, fileId)))
.getJson(Response.typeOf(CurseAddon.LatestFile.class));
return response.getData().toVersion().getFile();
} finally {
SEMAPHORE.release();
}
}
@Override
public Stream<RemoteMod.Version> getRemoteVersionsById(String id) throws IOException {
Response<List<CurseAddon.LatestFile>> response = withApiKey(HttpRequest.GET(PREFIX + "/v1/mods/" + id + "/files",
pair("pageSize", "10000")))
.getJson(Response.typeOf(listTypeOf(CurseAddon.LatestFile.class)));
return response.getData().stream().map(CurseAddon.LatestFile::toVersion);
}
public List<CurseAddon.Category> getCategoriesImpl() throws IOException {
Response<List<CurseAddon.Category>> categories = withApiKey(HttpRequest.GET(PREFIX + "/v1/categories", pair("gameId", "432")))
.getJson(Response.typeOf(listTypeOf(CurseAddon.Category.class)));
return reorganizeCategories(categories.getData(), section);
SEMAPHORE.acquireUninterruptibly();
try {
Response<List<CurseAddon.LatestFile>> response = withApiKey(HttpRequest.GET(PREFIX + "/v1/mods/" + id + "/files",
pair("pageSize", "10000")))
.getJson(Response.typeOf(listTypeOf(CurseAddon.LatestFile.class)));
return response.getData().stream().map(CurseAddon.LatestFile::toVersion);
} finally {
SEMAPHORE.release();
}
}
@Override
public Stream<RemoteModRepository.Category> getCategories() throws IOException {
return getCategoriesImpl().stream().map(CurseAddon.Category::toCategory);
SEMAPHORE.acquireUninterruptibly();
try {
Response<List<CurseAddon.Category>> categories = withApiKey(HttpRequest.GET(PREFIX + "/v1/categories", pair("gameId", "432")))
.getJson(Response.typeOf(listTypeOf(CurseAddon.Category.class)));
return reorganizeCategories(categories.getData(), section).stream().map(CurseAddon.Category::toCategory);
} finally {
SEMAPHORE.release();
}
}
private List<CurseAddon.Category> reorganizeCategories(List<CurseAddon.Category> categories, int rootId) {

100
HMCLCore/src/main/java/org/jackhuang/hmcl/mod/modrinth/ModrinthRemoteModRepository.java
View File

@@ -24,7 +24,11 @@ import org.jackhuang.hmcl.mod.LocalModFile;
import org.jackhuang.hmcl.mod.ModLoaderType;
import org.jackhuang.hmcl.mod.RemoteMod;
import org.jackhuang.hmcl.mod.RemoteModRepository;
import org.jackhuang.hmcl.util.*;
import org.jackhuang.hmcl.util.DigestUtils;
import org.jackhuang.hmcl.util.Immutable;
import org.jackhuang.hmcl.util.Lang;
import org.jackhuang.hmcl.util.Pair;
import org.jackhuang.hmcl.util.StringUtils;
import org.jackhuang.hmcl.util.gson.JsonUtils;
import org.jackhuang.hmcl.util.io.HttpRequest;
import org.jackhuang.hmcl.util.io.NetworkUtils;
@@ -35,7 +39,14 @@ import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.time.Instant;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Semaphore;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -49,6 +60,8 @@ public final class ModrinthRemoteModRepository implements RemoteModRepository {
public static final ModrinthRemoteModRepository RESOURCE_PACKS = new ModrinthRemoteModRepository("resourcepack");
public static final ModrinthRemoteModRepository SHADER_PACKS = new ModrinthRemoteModRepository("shader");
private static final Semaphore SEMAPHORE = new Semaphore(16);
private static final String PREFIX = "https://api.modrinth.com";
private final String projectType;
@@ -81,30 +94,36 @@ public final class ModrinthRemoteModRepository implements RemoteModRepository {
@Override
public SearchResult search(DownloadProvider downloadProvider, String gameVersion, @Nullable RemoteModRepository.Category category, int pageOffset, int pageSize, String searchFilter, SortType sort, SortOrder sortOrder) throws IOException {
List<List<String>> facets = new ArrayList<>();
facets.add(Collections.singletonList("project_type:" + projectType));
if (StringUtils.isNotBlank(gameVersion)) {
facets.add(Collections.singletonList("versions:" + gameVersion));
SEMAPHORE.acquireUninterruptibly();
try {
List<List<String>> facets = new ArrayList<>();
facets.add(Collections.singletonList("project_type:" + projectType));
if (StringUtils.isNotBlank(gameVersion)) {
facets.add(Collections.singletonList("versions:" + gameVersion));
}
if (category != null && StringUtils.isNotBlank(category.getId())) {
facets.add(Collections.singletonList("categories:" + category.getId()));
}
Map<String, String> query = mapOf(
pair("query", searchFilter),
pair("facets", JsonUtils.UGLY_GSON.toJson(facets)),
pair("offset", Integer.toString(pageOffset * pageSize)),
pair("limit", Integer.toString(pageSize)),
pair("index", convertSortType(sort))
);
Response<ProjectSearchResult> response = HttpRequest.GET(downloadProvider.injectURL(NetworkUtils.withQuery(PREFIX + "/v2/search", query)))
.getJson(Response.typeOf(ProjectSearchResult.class));
return new SearchResult(response.getHits().stream().map(ProjectSearchResult::toMod), (int) Math.ceil((double) response.totalHits / pageSize));
} finally {
SEMAPHORE.release();
}
if (category != null && StringUtils.isNotBlank(category.getId())) {
facets.add(Collections.singletonList("categories:" + category.getId()));
}
Map<String, String> query = mapOf(
pair("query", searchFilter),
pair("facets", JsonUtils.UGLY_GSON.toJson(facets)),
pair("offset", Integer.toString(pageOffset * pageSize)),
pair("limit", Integer.toString(pageSize)),
pair("index", convertSortType(sort))
);
Response<ProjectSearchResult> response = HttpRequest.GET(downloadProvider.injectURL(NetworkUtils.withQuery(PREFIX + "/v2/search", query)))
.getJson(Response.typeOf(ProjectSearchResult.class));
return new SearchResult(response.getHits().stream().map(ProjectSearchResult::toMod), (int) Math.ceil((double) response.totalHits / pageSize));
}
@Override
public Optional<RemoteMod.Version> getRemoteVersionByLocalFile(LocalModFile localModFile, Path file) throws IOException {
String sha1 = DigestUtils.digestToString("SHA-1", file);
SEMAPHORE.acquireUninterruptibly();
try {
ProjectVersion mod = HttpRequest.GET(PREFIX + "/v2/version_file/" + sha1,
pair("algorithm", "sha1"))
@@ -118,14 +137,21 @@ public final class ModrinthRemoteModRepository implements RemoteModRepository {
}
} catch (NoSuchFileException e) {
return Optional.empty();
} finally {
SEMAPHORE.release();
}
}
@Override
public RemoteMod getModById(String id) throws IOException {
id = StringUtils.removePrefix(id, "local-");
Project project = HttpRequest.GET(PREFIX + "/v2/project/" + id).getJson(Project.class);
return project.toMod();
SEMAPHORE.acquireUninterruptibly();
try {
id = StringUtils.removePrefix(id, "local-");
Project project = HttpRequest.GET(PREFIX + "/v2/project/" + id).getJson(Project.class);
return project.toMod();
} finally {
SEMAPHORE.release();
}
}
@Override
@@ -135,20 +161,28 @@ public final class ModrinthRemoteModRepository implements RemoteModRepository {
@Override
public Stream<RemoteMod.Version> getRemoteVersionsById(String id) throws IOException {
id = StringUtils.removePrefix(id, "local-");
List<ProjectVersion> versions = HttpRequest.GET(PREFIX + "/v2/project/" + id + "/version")
.getJson(listTypeOf(ProjectVersion.class));
return versions.stream().map(ProjectVersion::toVersion).flatMap(Lang::toStream);
}
public List<Category> getCategoriesImpl() throws IOException {
List<Category> categories = HttpRequest.GET(PREFIX + "/v2/tag/category").getJson(listTypeOf(Category.class));
return categories.stream().filter(category -> category.getProjectType().equals(projectType)).collect(Collectors.toList());
SEMAPHORE.acquireUninterruptibly();
try {
id = StringUtils.removePrefix(id, "local-");
List<ProjectVersion> versions = HttpRequest.GET(PREFIX + "/v2/project/" + id + "/version")
.getJson(listTypeOf(ProjectVersion.class));
return versions.stream().map(ProjectVersion::toVersion).flatMap(Lang::toStream);
} finally {
SEMAPHORE.release();
}
}
@Override
public Stream<RemoteModRepository.Category> getCategories() throws IOException {
return getCategoriesImpl().stream().map(Category::toCategory);
SEMAPHORE.acquireUninterruptibly();
try {
List<Category> categories = HttpRequest.GET(PREFIX + "/v2/tag/category").getJson(listTypeOf(Category.class));
return categories.stream()
.filter(category -> category.getProjectType().equals(projectType))
.map(Category::toCategory);
} finally {
SEMAPHORE.release();
}
}
public static class Category {
@@ -160,7 +194,7 @@ public final class ModrinthRemoteModRepository implements RemoteModRepository {
private final String projectType;
public Category() {
this("","","");
this("", "", "");
}
public Category(String icon, String name, String projectType) {

10
HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java
View File

@@ -509,12 +509,22 @@ public abstract class FetchTask<T> extends Task<T> {
return downloadExecutorConcurrency;
}
private static volatile boolean initialized = false;
public static void notifyInitialized() {
initialized = true;
}
/// Ensure that [#HTTP_CLIENT] is initialized after ProxyManager has been initialized.
private static final class Holder {
private static final HttpClient HTTP_CLIENT;
private static final String USER_AGENT = System.getProperty("http.agent", "HMCL");
static {
if (!initialized) {
throw new AssertionError("FetchTask.Holder accessed before ProxyManager initialization.");
}
boolean useHttp2 = !"false".equalsIgnoreCase(System.getProperty("hmcl.http2"));
HTTP_CLIENT = HttpClient.newBuilder()