IOUtils.java

Go to the documentation of this file.

/* *********************************************************************** *
 * project: org.matsim.*
 * IOUtils.java
 *                                                                         *
 * *********************************************************************** *
 *                                                                         *
 * copyright       : (C) 2007 by the members listed in the COPYING,        *
 *                   LICENSE and WARRANTY file.                            *
 * email           : info at matsim dot org                                *
 *                                                                         *
 * *********************************************************************** *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *   See also COPYING, LICENSE and WARRANTY file                           *
 *                                                                         *
 * *********************************************************************** */

package org.matsim.core.utils.io;

import com.github.luben.zstd.ZstdInputStream;
import com.github.luben.zstd.ZstdOutputStream;
import net.jpountz.lz4.LZ4FrameInputStream;
import net.jpountz.lz4.LZ4FrameOutputStream;
import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.UncheckedIOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.GeneralSecurityException;
import java.util.Arrays;
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

final public class IOUtils {

/*
The following says something about URLs vs Strings as file"names" (by Marcel):

Generell sollten File-Zugriffe entweder per URL oder per String erfolgen. Früher waren alles Strings, neu gibt es mit dem Config-Context auch URLs. Man sollte aber, wenn
man URLs verwendet, möglichst die ganze Aufruf-Kette als URL durchziehen, da es ansonsten Probleme geben kann (und eben nicht versuchen, die URL in einen String
umzuwandeln, wie ich in meiner Commit-Message geschrieben hatte).

Da wird eine URL erzeugt und in einen String umgewandelt, was definitiv zu Problemen führen wird, wenn der Pfad Leerzeichen enthält.

Zudem wird durch die Umwandlung in einen String dann im Code ein anderer Pfad aufgerufen (nämlich der für String-Pfade), welcher dann mit der verschachtelten URL für das File innerhalb eines Jars Probleme hat. In kurz:

        String urlString = "jar:file:/Users/kainagel/.m2/repository/org/matsim/matsim-examples/12.0-SNAPSHOT/matsim-examples-12.0-SNAPSHOT.jar!/test/scenarios/equil/config.xml”;
        URL url = new URL(urlString);
        InputStream is = new FileInputStream(new File(url.toURI()); // produces an exception, as it cannot access the file inside a Jar.
        InputStream is = url.openStream(); // this works
        InputStream is = new FileInputStream(new File(“file:/Users/kainagel/.m2/repository/org/matsim/matsim-examples/12.0-SNAPSHOT/matsim-examples-12.0-SNAPSHOT.jar"); // this works, as it is an actual file on the file system

In ConfigUtils.loadConfig( String[] ) wird nun IOUtils.resolveFileOrResource(String) aufgerufen, das unter der Annahme, dass weil das Argument ein String ist, es sich um ein File im Filesystem handeln muss, versucht mittels new File() darauf zuzugreifen. Da es sich aber eigentlich um eine URL handelt, die dazu noch ein File innerhalb eines Jar spezifiziert, schlägt das fehl.

Ich habe nun IOUtils.resolveFileOrResource(String) so angepasst, dass es versucht zu erkennen, ob das String-Argument eine URL ist, und in diesem Falle dies einfach als URL
zurückgibt und nicht noch all die anderen Tests durchführt, welche die Methode sonst macht.

Damit scheint dann auch dein Test zu funktionieren.

PR ist hier: https://github.com/matsim-org/matsim/pull/646
*/

    private IOUtils() {
    }

    private enum CompressionType { GZIP, LZ4, BZIP2, ZSTD }

    // Define compressions that can be used.
    private static final Map<String, CompressionType> COMPRESSION_EXTENSIONS = new TreeMap<>();

    static {
        COMPRESSION_EXTENSIONS.put("gz", CompressionType.GZIP);
        COMPRESSION_EXTENSIONS.put("lz4", CompressionType.LZ4);
        COMPRESSION_EXTENSIONS.put("bz2", CompressionType.BZIP2);
        COMPRESSION_EXTENSIONS.put("zst", CompressionType.ZSTD);
    }

    private static int zstdCompressionLevel = 6;

    public static void setZstdCompressionLevel(int level) {
        if (level >= 1) {
            zstdCompressionLevel = level;
        } else {
            logger.error("Invalid ZSTD compression level.");
        }
    }

    // Define a number of charsets that are / have been used.
    public static final Charset CHARSET_UTF8 = StandardCharsets.UTF_8;
    public static final Charset CHARSET_WINDOWS_ISO88591 = StandardCharsets.ISO_8859_1;

    // We niw use Unix line endings everywhere.
    public static final String NATIVE_NEWLINE = "\n";

    // Logger
    private final static Logger logger = LogManager.getLogger(IOUtils.class);

    public static URL resolveFileOrResource(String filename) throws UncheckedIOException {
        try {
            // I) do not handle URLs
            if (filename.startsWith("jar:file:") || filename.startsWith("file:") || filename.startsWith( "https:" ) || filename.startsWith( "http:" )) {
                // looks like an URI
                return new URL(filename);
            }

            // II) Replace home identifier
            if (filename.startsWith("~" + File.separator)) {
                filename = System.getProperty("user.home") + filename.substring(1);
            }

            // III.1) First, try to find the file in the file system
            File file = new File(filename);

            if (file.exists()) {
                logger.info(String.format("Resolved %s to %s", filename, file));
                return file.toURI().toURL();
            }

            // III.2) Try to find file with an additional postfix for compression
            for (String postfix : COMPRESSION_EXTENSIONS.keySet()) {
                file = new File(filename + "." + postfix);

                if (file.exists()) {
                    logger.info(String.format("Resolved %s to %s", filename, file));
                    return file.toURI().toURL();
                }
            }

            // IV.1) First, try to find the file in the class path
            URL resource = IOUtils.class.getClassLoader().getResource(filename);

            if (resource != null) {
                logger.info(String.format("Resolved %s to %s", filename, resource));
                return resource;
            }

            // IV.2) Second, try to find the resource with a compression extension
            for (String postfix : COMPRESSION_EXTENSIONS.keySet()) {
                resource = IOUtils.class.getClassLoader().getResource(filename + "." + postfix);

                if (resource != null) {
                    logger.info(String.format("Resolved %s to %s", filename, resource));
                    return resource;
                }
            }

            throw new FileNotFoundException(filename);
        } catch (FileNotFoundException | MalformedURLException e) {
            throw new UncheckedIOException(e);
        }
    }

    private static CompressionType getCompression(URL url) {

        // .enc extension is ignored
        String[] segments = url.getPath().replace(".enc", "").split("\\.");
        String lastExtension = segments[segments.length - 1];
        return COMPRESSION_EXTENSIONS.get(lastExtension.toLowerCase(Locale.ROOT));
    }

    public static InputStream getInputStream(URL url) throws UncheckedIOException {
        try {
            InputStream inputStream = url.openStream();

            if (url.getPath().endsWith(".enc"))
                inputStream = CipherUtils.getDecryptedInput(inputStream);

            CompressionType compression = getCompression(url);
            if (compression != null) {
                switch (compression) {
                    case GZIP:
                        inputStream = new GZIPInputStream(inputStream);
                        break;
                    case LZ4:
                        inputStream = new LZ4FrameInputStream(inputStream);
                        break;
                    case BZIP2:
                        inputStream = new CompressorStreamFactory().createCompressorInputStream(CompressorStreamFactory.BZIP2, inputStream);
                        break;
                    case ZSTD:
                        inputStream = new ZstdInputStream(inputStream);
                        break;
                }
            }

            return new UnicodeInputStream(new BufferedInputStream(inputStream));
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        } catch (CompressorException | GeneralSecurityException e) {
            throw new UncheckedIOException(new IOException(e));
        }
    }

    public static BufferedReader getBufferedReader(URL url, Charset charset) throws UncheckedIOException {
        InputStream inputStream = getInputStream(url);
        return new BufferedReader(new InputStreamReader(inputStream, charset));
    }

    public static BufferedReader getBufferedReader(URL url) throws UncheckedIOException {
        return getBufferedReader(url, CHARSET_UTF8);
    }

    public static OutputStream getOutputStream(URL url, boolean append) throws UncheckedIOException {
        try {
            if (!url.getProtocol().equals("file")) {
                throw new UncheckedIOException(new IOException("Can only write to file:// protocol URLs"));
            }

            File file = new File(url.toURI());
            CompressionType compression = getCompression(url);

            if ((compression != null && compression != CompressionType.ZSTD) && append && file.exists()) {
                throw new UncheckedIOException(new IOException("Cannot append to compressed files."));
            }

            OutputStream outputStream = new FileOutputStream(file, append);

            if (compression != null) {
                switch (compression) {
                    case GZIP:
                        outputStream = new GZIPOutputStream(outputStream);
                        break;
                    case LZ4:
                        outputStream = new LZ4FrameOutputStream(outputStream);
                        break;
                    case BZIP2:
                        outputStream = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.BZIP2, outputStream);
                        break;
                    case ZSTD:
                        outputStream = new ZstdOutputStream(outputStream, zstdCompressionLevel);
                        break;
                }
            }

            return new BufferedOutputStream(outputStream);
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        } catch (CompressorException | URISyntaxException e) {
            throw new UncheckedIOException(new IOException(e));
        }
    }

    public static BufferedWriter getBufferedWriter(URL url, Charset charset, boolean append)
            throws UncheckedIOException {
        OutputStream outputStream = getOutputStream(url, append);
        return new BufferedWriter(new OutputStreamWriter(outputStream, charset));
    }

    public static BufferedWriter getBufferedWriter(URL url) throws UncheckedIOException {
        return getBufferedWriter(url, CHARSET_UTF8, false);
    }

    public static PrintStream getPrintStream(URL url) throws UncheckedIOException {
        return new PrintStream(getOutputStream(url, false));
    }

    public static void copyStream(final InputStream fromStream, final OutputStream toStream)
            throws UncheckedIOException {
        try {
            byte[] buffer = new byte[4096];
            int bytesRead;

            while ((bytesRead = fromStream.read(buffer)) != -1) {
                toStream.write(buffer, 0, bytesRead);
            }
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    public static void deleteDirectoryRecursively(Path path) throws UncheckedIOException {
        try {
            Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
                @Override
                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                    Files.delete(file);
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                    Files.delete(dir);
                    return FileVisitResult.CONTINUE;
                }
            });
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    public static boolean isEqual(InputStream first, InputStream second) throws UncheckedIOException {
        byte[] buf1 = new byte[64 * 1024];
        byte[] buf2 = new byte[64 * 1024];
        try (first; second) {
            DataInputStream d2 = new DataInputStream(second);
            int len;
            while ((len = first.read(buf1)) > 0) {
                d2.readFully(buf2,0, len);
                if (!Arrays.equals(buf1, 0, len, buf2, 0, len)) {
                    return false;
                }
            }
            return d2.read() < 0; // is the end of the second file also.
        } catch(EOFException ioe) {
            return false;
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    public static URL getFileUrl(String filename) throws UncheckedIOException {
        try {
            return new File(filename).toURI().toURL();
        } catch (MalformedURLException e) {
            throw new UncheckedIOException(e);
        }
    }

    public static URL extendUrl(URL context, String extension) throws UncheckedIOException {
        if (context == null) {
            throw new IllegalArgumentException("Please use IOUtils.getFileUrl");
        }

        try {
            return new URL(context, extension);
        } catch (MalformedURLException e) {
            // We cannot construct a URL for some reason (see respective unit test)
            return getFileUrl(extension);
        }
    }

    public static BufferedReader getBufferedReader(String filename) {
        return getBufferedReader(resolveFileOrResource(filename));
    }

    public static BufferedWriter getBufferedWriter(String filename) {
        return getBufferedWriter(getFileUrl(filename));
    }

    public static BufferedWriter getAppendingBufferedWriter(String filename) {
        return getBufferedWriter(getFileUrl(filename), CHARSET_UTF8, true);
    }

    public static void copyFile(String inputFilename, String outputFilename) throws IOException {
        File fromFile = new File(inputFilename);
        File toFile = new File(outputFilename);
        Files.copy(fromFile.toPath(), toFile.toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
    }
}