Auto detect encoding of compressed packs

This commit is contained in:
huanghongxun
2018-12-13 00:42:52 +08:00
parent 2cb23cbfcb
commit f5ffa875b8
21 changed files with 294 additions and 118 deletions

View File

@@ -42,6 +42,7 @@ public final class CurseInstallTask extends Task {
private final DefaultDependencyManager dependencyManager;
private final DefaultGameRepository repository;
private final File zipFile;
private final Modpack modpack;
private final CurseManifest manifest;
private final String name;
private final File run;
@@ -58,9 +59,10 @@ public final class CurseInstallTask extends Task {
* @param name the new version name
* @see CurseManifest#readCurseForgeModpackManifest
*/
public CurseInstallTask(DefaultDependencyManager dependencyManager, File zipFile, CurseManifest manifest, String name) {
public CurseInstallTask(DefaultDependencyManager dependencyManager, File zipFile, Modpack modpack, CurseManifest manifest, String name) {
this.dependencyManager = dependencyManager;
this.zipFile = zipFile;
this.modpack = modpack;
this.manifest = manifest;
this.name = name;
this.repository = dependencyManager.getGameRepository();
@@ -92,7 +94,7 @@ public final class CurseInstallTask extends Task {
} catch (JsonParseException | IOException ignore) {
}
this.config = config;
dependents.add(new ModpackInstallTask<>(zipFile, run, manifest.getOverrides(), any -> true, config));
dependents.add(new ModpackInstallTask<>(zipFile, run, modpack.getEncoding(), manifest.getOverrides(), any -> true, config));
}
@Override
@@ -121,7 +123,7 @@ public final class CurseInstallTask extends Task {
FileUtils.writeText(new File(root, "manifest.json"), JsonUtils.GSON.toJson(manifest));
dependencies.add(new CurseCompletionTask(dependencyManager, name, manifest));
dependencies.add(new MinecraftInstanceTask<>(zipFile, manifest.getOverrides(), manifest, MODPACK_TYPE, repository.getModpackConfiguration(name)));
dependencies.add(new MinecraftInstanceTask<>(zipFile, modpack.getEncoding(), manifest.getOverrides(), manifest, MODPACK_TYPE, repository.getModpackConfiguration(name)));
}
public static final String MODPACK_TYPE = "Curse";

View File

@@ -19,13 +19,13 @@ package org.jackhuang.hmcl.mod;
import com.google.gson.JsonParseException;
import com.google.gson.annotations.SerializedName;
import org.jackhuang.hmcl.util.Immutable;
import org.jackhuang.hmcl.util.gson.JsonUtils;
import org.jackhuang.hmcl.util.io.CompressingUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
@@ -117,11 +117,11 @@ public final class CurseManifest {
* @throws JsonParseException if the manifest.json is missing or malformed.
* @return the manifest.
*/
public static Modpack readCurseForgeModpackManifest(File f) throws IOException, JsonParseException {
String json = CompressingUtils.readTextZipEntry(f, "manifest.json");
public static Modpack readCurseForgeModpackManifest(Path zip, Charset encoding) throws IOException, JsonParseException {
String json = CompressingUtils.readTextZipEntry(zip, "manifest.json", encoding);
CurseManifest manifest = JsonUtils.fromNonNullJson(json, CurseManifest.class);
return new Modpack(manifest.getName(), manifest.getAuthor(), manifest.getVersion(), manifest.getMinecraft().getGameVersion(),
CompressingUtils.readTextZipEntryQuietly(f, "modlist.html").orElse( "No description"), manifest);
CompressingUtils.readTextZipEntryQuietly(zip, "modlist.html", encoding).orElse( "No description"), encoding, manifest);
}
public static final String MINECRAFT_MODPACK = "minecraftModpack";

View File

@@ -24,6 +24,7 @@ import org.jackhuang.hmcl.util.io.FileUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.LinkedList;
@@ -35,13 +36,15 @@ import static org.jackhuang.hmcl.util.Hex.encodeHex;
public final class MinecraftInstanceTask<T> extends Task {
private final File zipFile;
private final Charset encoding;
private final String subDirectory;
private final File jsonFile;
private final T manifest;
private final String type;
public MinecraftInstanceTask(File zipFile, String subDirectory, T manifest, String type, File jsonFile) {
public MinecraftInstanceTask(File zipFile, Charset encoding, String subDirectory, T manifest, String type, File jsonFile) {
this.zipFile = zipFile;
this.encoding = encoding;
this.subDirectory = FileUtils.normalizePath(subDirectory);
this.manifest = manifest;
this.jsonFile = jsonFile;
@@ -55,7 +58,7 @@ public final class MinecraftInstanceTask<T> extends Task {
public void execute() throws Exception {
List<ModpackConfiguration.FileInformation> overrides = new LinkedList<>();
try (FileSystem fs = CompressingUtils.createReadOnlyZipFileSystem(zipFile.toPath())) {
try (FileSystem fs = CompressingUtils.readonly(zipFile.toPath()).setEncoding(encoding).build()) {
Path root = fs.getPath(subDirectory);
if (Files.exists(root))

View File

@@ -17,6 +17,8 @@
*/
package org.jackhuang.hmcl.mod;
import java.nio.charset.Charset;
/**
*
* @author huangyuhui
@@ -27,18 +29,20 @@ public final class Modpack {
private final String version;
private final String gameVersion;
private final String description;
private final transient Charset encoding;
private final Object manifest;
public Modpack() {
this("", null, null, null, null, null);
this("", null, null, null, null, null, null);
}
public Modpack(String name, String author, String version, String gameVersion, String description, Object manifest) {
public Modpack(String name, String author, String version, String gameVersion, String description, Charset encoding, Object manifest) {
this.name = name;
this.author = author;
this.version = version;
this.gameVersion = gameVersion;
this.description = description;
this.encoding = encoding;
this.manifest = manifest;
}
@@ -59,18 +63,26 @@ public final class Modpack {
}
public Modpack setGameVersion(String gameVersion) {
return new Modpack(name, author, version, gameVersion, description, manifest);
return new Modpack(name, author, version, gameVersion, description, encoding, manifest);
}
public String getDescription() {
return description;
}
public Charset getEncoding() {
return encoding;
}
public Modpack setEncoding(Charset encoding) {
return new Modpack(name, author, version, gameVersion, description, encoding, manifest);
}
public Object getManifest() {
return manifest;
}
public Modpack setManifest(Object manifest) {
return new Modpack(name, author, version, gameVersion, description, manifest);
return new Modpack(name, author, version, gameVersion, description, encoding, manifest);
}
}

View File

@@ -23,6 +23,7 @@ import org.jackhuang.hmcl.util.io.Unzipper;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.*;
import java.util.function.Predicate;
@@ -34,13 +35,15 @@ public class ModpackInstallTask<T> extends Task {
private final File modpackFile;
private final File dest;
private final Charset charset;
private final String subDirectory;
private final List<ModpackConfiguration.FileInformation> overrides;
private final Predicate<String> callback;
public ModpackInstallTask(File modpackFile, File dest, String subDirectory, Predicate<String> callback, ModpackConfiguration<T> oldConfiguration) {
public ModpackInstallTask(File modpackFile, File dest, Charset charset, String subDirectory, Predicate<String> callback, ModpackConfiguration<T> oldConfiguration) {
this.modpackFile = modpackFile;
this.dest = dest;
this.charset = charset;
this.subDirectory = subDirectory;
this.callback = callback;
@@ -64,6 +67,7 @@ public class ModpackInstallTask<T> extends Task {
.setSubDirectory(subDirectory)
.setTerminateIfSubDirectoryNotExists()
.setReplaceExistentFile(true)
.setEncoding(charset)
.setFilter((destPath, isDirectory, zipEntry, entryPath) -> {
if (isDirectory) return true;
if (!callback.test(entryPath)) return false;

View File

@@ -21,9 +21,9 @@ import org.jackhuang.hmcl.util.Lang;
import org.jackhuang.hmcl.util.StringUtils;
import org.jackhuang.hmcl.util.io.CompressingUtils;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -264,9 +264,9 @@ public final class MultiMCInstanceConfiguration {
return mmcPack;
}
public static Modpack readMultiMCModpackManifest(File modpackFile) throws IOException {
MultiMCManifest manifest = MultiMCManifest.readMultiMCModpackManifest(modpackFile);
try (FileSystem fs = CompressingUtils.createReadOnlyZipFileSystem(modpackFile.toPath())) {
public static Modpack readMultiMCModpackManifest(Path modpackFile, Charset encoding) throws IOException {
MultiMCManifest manifest = MultiMCManifest.readMultiMCModpackManifest(modpackFile, encoding);
try (FileSystem fs = CompressingUtils.readonly(modpackFile).setEncoding(encoding).build()) {
Path root = Files.list(fs.getPath("/")).filter(Files::isDirectory).findAny()
.orElseThrow(() -> new IOException("Not a valid MultiMC modpack"));
String name = StringUtils.removeSuffix(root.normalize().getFileName().toString(), "/");
@@ -275,7 +275,7 @@ public final class MultiMCInstanceConfiguration {
if (Files.notExists(instancePath))
throw new IOException("`instance.cfg` not found, " + modpackFile + " is not a valid MultiMC modpack.");
MultiMCInstanceConfiguration cfg = new MultiMCInstanceConfiguration(name, Files.newInputStream(instancePath), manifest);
return new Modpack(cfg.getName(), "", "", cfg.getGameVersion(), cfg.getNotes(), cfg);
return new Modpack(cfg.getName(), "", "", cfg.getGameVersion(), cfg.getNotes(), encoding, cfg);
}
}
}

View File

@@ -18,14 +18,13 @@
package org.jackhuang.hmcl.mod;
import com.google.gson.annotations.SerializedName;
import org.jackhuang.hmcl.util.Immutable;
import org.jackhuang.hmcl.util.gson.JsonUtils;
import org.jackhuang.hmcl.util.io.CompressingUtils;
import org.jackhuang.hmcl.util.io.IOUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -53,8 +52,8 @@ public final class MultiMCManifest {
return components;
}
public static MultiMCManifest readMultiMCModpackManifest(File zipFile) throws IOException {
try (FileSystem fs = CompressingUtils.createReadOnlyZipFileSystem(zipFile.toPath())) {
public static MultiMCManifest readMultiMCModpackManifest(Path zipFile, Charset encoding) throws IOException {
try (FileSystem fs = CompressingUtils.readonly(zipFile).setEncoding(encoding).build()) {
Path root = Files.list(fs.getPath("/")).filter(Files::isDirectory).findAny()
.orElseThrow(() -> new IOException("Not a valid MultiMC modpack"));
Path mmcPack = root.resolve("mmc-pack.json");

View File

@@ -26,7 +26,7 @@ import org.jackhuang.hmcl.game.Arguments;
import org.jackhuang.hmcl.game.DefaultGameRepository;
import org.jackhuang.hmcl.game.Version;
import org.jackhuang.hmcl.task.Task;
import org.jackhuang.hmcl.util.*;
import org.jackhuang.hmcl.util.Lang;
import org.jackhuang.hmcl.util.gson.JsonUtils;
import org.jackhuang.hmcl.util.io.CompressingUtils;
import org.jackhuang.hmcl.util.io.FileUtils;
@@ -49,14 +49,16 @@ import java.util.Optional;
public final class MultiMCModpackInstallTask extends Task {
private final File zipFile;
private final Modpack modpack;
private final MultiMCInstanceConfiguration manifest;
private final String name;
private final DefaultGameRepository repository;
private final List<Task> dependencies = new LinkedList<>();
private final List<Task> dependents = new LinkedList<>();
public MultiMCModpackInstallTask(DefaultDependencyManager dependencyManager, File zipFile, MultiMCInstanceConfiguration manifest, String name) {
public MultiMCModpackInstallTask(DefaultDependencyManager dependencyManager, File zipFile, Modpack modpack, MultiMCInstanceConfiguration manifest, String name) {
this.zipFile = zipFile;
this.modpack = modpack;
this.manifest = manifest;
this.name = name;
this.repository = dependencyManager.getGameRepository();
@@ -100,7 +102,7 @@ public final class MultiMCModpackInstallTask extends Task {
} catch (JsonParseException | IOException ignore) {
}
dependents.add(new ModpackInstallTask<>(zipFile, run, "/" + manifest.getName() + "/minecraft", any -> true, config));
dependents.add(new ModpackInstallTask<>(zipFile, run, modpack.getEncoding(), "/" + manifest.getName() + "/minecraft", any -> true, config));
}
@Override
@@ -141,7 +143,7 @@ public final class MultiMCModpackInstallTask extends Task {
}
dependencies.add(new VersionJsonSaveTask(repository, version));
dependencies.add(new MinecraftInstanceTask<>(zipFile, "/" + manifest.getName() + "/minecraft", manifest, MODPACK_TYPE, repository.getModpackConfiguration(name)));
dependencies.add(new MinecraftInstanceTask<>(zipFile, modpack.getEncoding(), "/" + manifest.getName() + "/minecraft", manifest, MODPACK_TYPE, repository.getModpackConfiguration(name)));
}
public static final String MODPACK_TYPE = "MultiMC";

View File

@@ -23,7 +23,10 @@ import javafx.beans.property.ReadOnlyDoubleWrapper;
import javafx.beans.property.ReadOnlyStringProperty;
import javafx.beans.property.ReadOnlyStringWrapper;
import org.jackhuang.hmcl.event.EventManager;
import org.jackhuang.hmcl.util.*;
import org.jackhuang.hmcl.util.AutoTypingMap;
import org.jackhuang.hmcl.util.InvocationDispatcher;
import org.jackhuang.hmcl.util.Logging;
import org.jackhuang.hmcl.util.ReflectionHelper;
import org.jackhuang.hmcl.util.function.ExceptionalConsumer;
import org.jackhuang.hmcl.util.function.ExceptionalFunction;
import org.jackhuang.hmcl.util.function.ExceptionalRunnable;
@@ -31,7 +34,6 @@ import org.jackhuang.hmcl.util.function.ExceptionalRunnable;
import java.util.Collection;
import java.util.Collections;
import java.util.concurrent.Callable;
import java.util.function.Function;
import java.util.logging.Level;
/**
@@ -363,7 +365,7 @@ public abstract class Task {
return new TaskCallable<>(id, callable);
}
public static <V> TaskResult<V> ofResult(String id, Function<AutoTypingMap<String>, V> closure) {
public static <V> TaskResult<V> ofResult(String id, ExceptionalFunction<AutoTypingMap<String>, V, ?> closure) {
return new TaskCallable2<>(id, closure);
}

View File

@@ -18,8 +18,7 @@
package org.jackhuang.hmcl.task;
import org.jackhuang.hmcl.util.AutoTypingMap;
import java.util.function.Function;
import org.jackhuang.hmcl.util.function.ExceptionalFunction;
/**
*
@@ -28,9 +27,9 @@ import java.util.function.Function;
class TaskCallable2<V> extends TaskResult<V> {
private final String id;
private final Function<AutoTypingMap<String>, V> callable;
private final ExceptionalFunction<AutoTypingMap<String>, V, ?> callable;
public TaskCallable2(String id, Function<AutoTypingMap<String>, V> callable) {
public TaskCallable2(String id, ExceptionalFunction<AutoTypingMap<String>, V, ?> callable) {
this.id = id;
this.callable = callable;
}
@@ -41,7 +40,7 @@ class TaskCallable2<V> extends TaskResult<V> {
}
@Override
public void execute() {
public void execute() throws Exception {
setResult(callable.apply(getVariables()));
}
}

View File

@@ -17,14 +17,20 @@
*/
package org.jackhuang.hmcl.util.io;
import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.Path;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.ZipError;
import java.util.zip.ZipException;
@@ -43,28 +49,127 @@ public final class CompressingUtils {
private CompressingUtils() {
}
@NotNull
private static FileVisitResult testZipPath(Path file, Path root, AtomicBoolean result) {
try {
root.relativize(file).toString(); // throw IllegalArgumentException for wrong encoding.
return FileVisitResult.CONTINUE;
} catch (Exception e) {
result.set(false);
return FileVisitResult.TERMINATE;
}
}
public static boolean testEncoding(Path zipFile, Charset encoding) throws IOException {
AtomicBoolean result = new AtomicBoolean(true);
try (FileSystem fs = CompressingUtils.createReadOnlyZipFileSystem(zipFile, encoding)) {
Path root = fs.getPath("/");
Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file,
BasicFileAttributes attrs) {
return testZipPath(file, root, result);
}
@Override
public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) {
return testZipPath(dir, root, result);
}
});
}
return result.get();
}
public static Charset findSuitableEncoding(Path zipFile) throws IOException {
return findSuitableEncoding(zipFile, Charset.availableCharsets().values());
}
public static Charset findSuitableEncoding(Path zipFile, Collection<Charset> candidates) throws IOException {
if (testEncoding(zipFile, StandardCharsets.UTF_8)) return StandardCharsets.UTF_8;
if (testEncoding(zipFile, Charset.defaultCharset())) return Charset.defaultCharset();
for (Charset charset : candidates)
if (charset != null && testEncoding(zipFile, charset))
return charset;
throw new IOException("Cannot find suitable encoding for the zip.");
}
public static final class Builder {
private boolean autoDetectEncoding = false;
private Collection<Charset> charsetCandidates;
private Charset encoding = StandardCharsets.UTF_8;
private boolean useTempFile = false;
private final boolean create;
private final Path zip;
public Builder(Path zip, boolean create) {
this.zip = zip;
this.create = create;
}
public Builder setAutoDetectEncoding(boolean autoDetectEncoding) {
this.autoDetectEncoding = autoDetectEncoding;
return this;
}
public Builder setCharsetCandidates(Collection<Charset> charsetCandidates) {
this.charsetCandidates = charsetCandidates;
return this;
}
public Builder setEncoding(Charset encoding) {
this.encoding = encoding;
return this;
}
public Builder setUseTempFile(boolean useTempFile) {
this.useTempFile = useTempFile;
return this;
}
public FileSystem build() throws IOException {
if (autoDetectEncoding) {
if (!testEncoding(zip, encoding)) {
if (charsetCandidates == null)
charsetCandidates = Charset.availableCharsets().values();
encoding = findSuitableEncoding(zip, charsetCandidates);
}
}
return createZipFileSystem(zip, create, useTempFile, encoding);
}
}
public static Builder readonly(Path zipFile) {
return new Builder(zipFile, false);
}
public static Builder writable(Path zipFile) {
return new Builder(zipFile, true).setUseTempFile(true);
}
public static FileSystem createReadOnlyZipFileSystem(Path zipFile) throws IOException {
return createReadOnlyZipFileSystem(zipFile, null);
}
public static FileSystem createReadOnlyZipFileSystem(Path zipFile, String encoding) throws IOException {
return createZipFileSystem(zipFile, false, false, encoding);
public static FileSystem createReadOnlyZipFileSystem(Path zipFile, Charset charset) throws IOException {
return createZipFileSystem(zipFile, false, false, charset);
}
public static FileSystem createWritableZipFileSystem(Path zipFile) throws IOException {
return createWritableZipFileSystem(zipFile, null);
}
public static FileSystem createWritableZipFileSystem(Path zipFile, String encoding) throws IOException {
return createZipFileSystem(zipFile, true, true, encoding);
public static FileSystem createWritableZipFileSystem(Path zipFile, Charset charset) throws IOException {
return createZipFileSystem(zipFile, true, true, charset);
}
public static FileSystem createZipFileSystem(Path zipFile, boolean create, boolean useTempFile, String encoding) throws IOException {
public static FileSystem createZipFileSystem(Path zipFile, boolean create, boolean useTempFile, Charset encoding) throws IOException {
Map<String, Object> env = new HashMap<>();
if (create)
env.put("create", "true");
if (encoding != null)
env.put("encoding", encoding);
env.put("encoding", encoding.name());
if (useTempFile)
env.put("useTempFile", true);
try {
@@ -86,7 +191,19 @@ public final class CompressingUtils {
* @return the plain text content of given file.
*/
public static String readTextZipEntry(File zipFile, String name) throws IOException {
try (FileSystem fs = createReadOnlyZipFileSystem(zipFile.toPath())) {
return readTextZipEntry(zipFile.toPath(), name, null);
}
/**
* Read the text content of a file in zip.
*
* @param zipFile the zip file
* @param name the location of the text in zip file, something like A/B/C/D.txt
* @throws IOException if the file is not a valid zip file.
* @return the plain text content of given file.
*/
public static String readTextZipEntry(Path zipFile, String name, Charset encoding) throws IOException {
try (FileSystem fs = createReadOnlyZipFileSystem(zipFile, encoding)) {
return FileUtils.readText(fs.getPath(name));
}
}
@@ -105,4 +222,19 @@ public final class CompressingUtils {
return Optional.empty();
}
}
/**
* Read the text content of a file in zip.
*
* @param file the zip file
* @param name the location of the text in zip file, something like A/B/C/D.txt
* @return the plain text content of given file.
*/
public static Optional<String> readTextZipEntryQuietly(Path file, String name, Charset encoding) {
try {
return Optional.of(readTextZipEntry(file, name, encoding));
} catch (IOException e) {
return Optional.empty();
}
}
}

View File

@@ -19,6 +19,8 @@ package org.jackhuang.hmcl.util.io;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
@@ -28,7 +30,7 @@ public class Unzipper {
private boolean terminateIfSubDirectoryNotExists = false;
private String subDirectory = "/";
private FileFilter filter = null;
private String encoding;
private Charset encoding = StandardCharsets.UTF_8;
/**
* Decompress the given zip file to a directory.
@@ -82,7 +84,7 @@ public class Unzipper {
return this;
}
public Unzipper setEncoding(String encoding) {
public Unzipper setEncoding(Charset encoding) {
this.encoding = encoding;
return this;
}
@@ -99,7 +101,7 @@ public class Unzipper {
*/
public void unzip() throws IOException {
Files.createDirectories(dest);
try (FileSystem fs = CompressingUtils.createReadOnlyZipFileSystem(zipFile, encoding)) {
try (FileSystem fs = CompressingUtils.readonly(zipFile).setEncoding(encoding).setAutoDetectEncoding(true).build()) {
Path root = fs.getPath(subDirectory);
if (!root.isAbsolute() || (subDirectory.length() > 1 && subDirectory.endsWith("/")))
throw new IllegalArgumentException("Subdirectory for unzipper must be absolute");

View File

@@ -22,6 +22,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.function.Predicate;
@@ -39,7 +40,7 @@ public final class Zipper implements Closeable {
this(zipFile, null);
}
public Zipper(Path zipFile, String encoding) throws IOException {
public Zipper(Path zipFile, Charset encoding) throws IOException {
Files.deleteIfExists(zipFile);
fs = CompressingUtils.createWritableZipFileSystem(zipFile, encoding);
}