/*
 * Copyright 2007-2009 Medsea Business Solutions S.L.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package eu.medsea.mimeutil.detector;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import eu.medsea.mimeutil.MimeException;
import eu.medsea.mimeutil.MimeUtil;

/**
 * The magic mime rules files are loaded in the following way.
 * <ol>
 * <li>From a JVM system property <code>magic-mime</code> i.e
 * <code>-Dmagic-mime=../my/magic/mime/rules</code></li>
 * <li>From any file named <code>magic.mime</code> that can be found on the
 * classpath</li>
 * <li>From a file named <code>.magic.mime</code> in the users home directory</li>
 * <li>From the normal Unix locations <code>/usr/share/file/magic.mime</code>
 * and <code>/etc/magic.mime</code> (in that order)</li>
 * <li>From the internal <code>magic.mime</code> file
 * <code>eu.medsea.mimeutil.magic.mime</code> if, and only if, no files are
 * located in step 4 above.</li>
 * </ol>
 * Each rule file is appended to the end of the existing rules so the earlier in
 * the sequence you define a rule means this will take precedence over rules
 * loaded later. </p>
 * <p>
 * You can add new mime mapping rules using the syntax defined for the Unix
 * magic.mime file by placing these rules in any of the files or locations
 * listed above. You can also change an existing mapping rule by redefining the
 * existing rule in one of the files listed above. This is handy for some of the
 * more sketchy rules defined in the existing Unix magic.mime files.
 * <p>
 * We extended the string type rule which allows you to match strings in a file
 * where you do not know the actual offset of the string containing magic file
 * information it goes something like “what I am looking for will be ‘somewhere’
 * within the next n characters” from this location. This is an important
 * improvement to the string matching rules especially for text based documents
 * such as HTML and XML formats. The reasoning for this was that the rules for
 * matching SVG images defined in the original 'magic.mime' file hardly ever
 * worked, this is because of the fixed offset definitions within the magic rule
 * format. As XML documents generally have an XML declaration that can contain
 * various optional attributes the length of this header often cannot be
 * determined, therefore we cannot know that the DOCTYPE declaration for an SVG
 * xml file starts at “this” location, all we can say is that, if this is an SVG
 * xml file then it will have an SVG DOCTYPE somewhere near the beginning of the
 * file and probably within the first 1024 characters. So we test for the xml
 * declaration and then we test for the DOCTYPE within a specified number of
 * characters and if found then we match this rule. This extension can be used
 * to better identify ALL of the XML type mime mappings in the current
 * 'magic.mime' file. Remember though, as we stated earlier mime type matching
 * using any of the mechanisms supported is not an exact science and should
 * always be viewed as a 'best guess' and not as a 'definite match'.
 * </p>
 * <p>
 * An example of overriding the PNG and SVG rules can be found in our internal
 * 'magic.mime' file located in the test_files directory (this file is NOT used
 * when locating rules and is used for testing purposes only). This PNG rule
 * overrides the original PNG rule defined in the 'magic.mime' file we took from
 * the Internet, and the SVG rule overrides the SVG detection also defined in
 * the original 'magic.mime' file
 * </p>
 * <p>
 *
 * <pre>
 * #PNG Image Format
 * 0		string		\211PNG\r\n\032\n		image/png
 *
 * #SVG Image Format
 * #	We know its an XML file so it should start with an XML declaration.
 * 0	string	\&lt;?xml\ version=	text/xml
 * #	As the XML declaration in an XML file can be short or extended we cannot know
 * #	exactly where the declaration ends i.e. how long it is,
 * #	also it could be terminated by a new line(s) or a space(s).
 * #	So the next line states that somewhere after the 15th character position we should find the DOCTYPE declaration.
 * #	This DOCTYPE declaration should be within 1024 characters from the 15th character
 * &gt;15	string&gt;1024&lt;	\&lt;!DOCTYPE\ svg\ PUBLIC\ &quot;-//W3C//DTD\ SVG 	image/svg+xml
 * </pre>
 *
 * </p>
 * <p>
 * As you can see the extension is defined using the syntax string>bufsize<. It
 * can only be used on a string type and basically means match this within
 * bufsize character from the position defined at the beginning of the line.
 * This rule is much more verbose than required as we really only need to check
 * for the presence of SVG. As we said earlier, this is a test case file and not
 * used by the utility under normal circumstances.
 *
 * The test mime-types.properties and magic.mime files we use can be located in
 * the test_files directory of this distribution.
 * </p>
 * <p>
 * We use the <code>application/directory</code> mime type to identify
 * directories. Even though this is not an official mime type it seems to be
 * well accepted on the net as an unofficial mime type so we thought it was OK
 * for us to use as well.
 * </p>
 * <p>
 * This class is auto loaded by MimeUtil as it has an entry in the file called
 * MimeDetectors. MimeUtil reads this file at startup and calls Class.forName()
 * on each entry found. This mean the MimeDetector must have a no arg
 * constructor.
 * </p>
 *
 * @author Steven McArdle.
 *
 */
public class MagicMimeMimeDetector extends MimeDetector {

	private static Logger log = LoggerFactory.getLogger(MagicMimeMimeDetector.class);

	// Having the defaultLocations as protected allows you to subclass this class
	// and add different paths or remove them all so that the internal file is always used
	protected static String[] defaultLocations = { "/usr/share/mimelnk/magic",
			"/usr/share/file/magic.mime", "/etc/magic.mime" };
	private static List magicMimeFileLocations = Arrays
			.asList(defaultLocations);

	private static ArrayList mMagicMimeEntries = new ArrayList();

	public MagicMimeMimeDetector() {
		MagicMimeMimeDetector.initMagicRules();
	}

	public String getDescription() {
		return "Get the mime types of files or streams using the Unix file(5) magic.mime files";
	}

	/**
	 * Get the mime types that may be contained in the data array.
	 *
	 * @param data. The byte array that contains data we want to detect mime types from.
	 * @return the mime types.
	 * @throws MimeException if for instance we try to match beyond the end of the data.
	 */
	public Collection getMimeTypesByteArray(final byte[] data)
			throws UnsupportedOperationException {
		Collection mimeTypes = new LinkedHashSet();
		int len = mMagicMimeEntries.size();
		try {
			for (int i = 0; i < len; i++) {
				MagicMimeEntry me = (MagicMimeEntry) mMagicMimeEntries.get(i);
				MagicMimeEntry matchingMagicMimeEntry = me.getMatch(data);
				if (matchingMagicMimeEntry != null) {
					mimeTypes.add(matchingMagicMimeEntry.getMimeType());
				}
			}
		} catch (Exception e) {
			log.error(e.getMessage(), e);
		}
		return mimeTypes;
	}


	/**
	 * Get the mime types of the data in the specified {@link InputStream}.
	 * Therefore, the <code>InputStream</code> must support mark and reset (see
	 * {@link InputStream#markSupported()}). If it does not support mark and
	 * reset, an {@link MimeException} is thrown.
	 *
	 * @param in
	 *            the stream from which to read the data.
	 * @return the mime types.
	 * @throws MimeException
	 *             if the specified <code>InputStream</code> does not support
	 *             mark and reset (see {@link InputStream#markSupported()}).
	 */
	public Collection getMimeTypesInputStream(final InputStream in)
			throws UnsupportedOperationException {
		Collection mimeTypes = new LinkedHashSet();
		int len = mMagicMimeEntries.size();
		try {
			for (int i = 0; i < len; i++) {
				MagicMimeEntry me = (MagicMimeEntry) mMagicMimeEntries.get(i);
				MagicMimeEntry matchingMagicMimeEntry = me.getMatch(in);
				if (matchingMagicMimeEntry != null) {
					mimeTypes.add(matchingMagicMimeEntry.getMimeType());
				}
			}
		} catch (Exception e) {
			log.error(e.getMessage(), e);
		}
		return mimeTypes;
	}

	/**
	 * Defer this call to the File method
	 */
	public Collection getMimeTypesFileName(final String fileName) throws UnsupportedOperationException {
		return getMimeTypesFile(new File(fileName));
	}


	/**
	 * Defer this call to the InputStream method
	 */
	public Collection getMimeTypesURL(final URL url) throws UnsupportedOperationException {
		InputStream in = null;
		try {
			return getMimeTypesInputStream(in = new BufferedInputStream(MimeUtil.getInputStreamForURL(url)));
		}catch(Exception e) {
			throw new MimeException(e);
		}finally {
			closeStream(in);
		}
	}

	/**
	 * Defer this call to the InputStream method
	 */
	public Collection getMimeTypesFile(final File file) throws UnsupportedOperationException {
		InputStream in = null;
		try {
			return getMimeTypesInputStream(in = new BufferedInputStream(new FileInputStream(file)));
		}catch(FileNotFoundException e) {
			throw new UnsupportedOperationException(e.getLocalizedMessage());
		}catch(Exception e) {
			throw new MimeException(e);
		}finally {
			closeStream(in);
		}
	}

	/*
	 * This loads the magic.mime file rules into the internal parse tree in the
	 * following order 1. From any magic.mime that can be located on the
	 * classpath 2. From any magic.mime file that can be located using the
	 * environment variable MAGIC 3. From any magic.mime located in the users
	 * home directory ~/.magic.mime file if the MAGIC environment variable is
	 * not set 4. From the locations defined in the magicMimeFileLocations and
	 * the order defined 5. From the internally defined magic.mime file ONLY if
	 * we are unable to locate any of the files in steps 2 - 5 above Thanks go
	 * to Simon Pepping for his bug report
	 */
	private static void initMagicRules() {
		InputStream in = null;

		// Try to locate a magic.mime file locate by system property magic-mime
		try {
			String fname = System.getProperty("magic-mime");
			if (fname != null && fname.length() != 0) {
				in = new FileInputStream(fname);
				if (in != null) {
					parse("-Dmagic-mime=" + fname, new InputStreamReader(in));
				}
			}
		} catch (Exception e) {
			log.error("Failed to parse custom magic mime file defined by system property -Dmagic-mime ["
					+ System.getProperty("magic-mime")
					+ "]. File will be ignored.", e);
		} finally {
			in = closeStream(in);
		}

		// Try to locate a magic.mime file(s) on the classpath

		// Get an enumeration of all files on the classpath with this name. They could be in jar files as well
		try {
			Enumeration en = MimeUtil.class.getClassLoader().getResources("magic.mime");
			while(en.hasMoreElements()) {
				URL url = (URL)en.nextElement();
				in = url.openStream();
				if(in != null) {
					try {
						parse("classpath:[" + url + "]", new InputStreamReader(in));
					} catch(Exception ex) {
						log.error("Failed to parse magic.mime rule file [" + url + "] on the classpath. File will be ignored.",
							ex);
					}
				}

			}
		}catch(Exception e) {
			log.error("Problem while processing magic.mime files from classpath. Files will be ignored.", e);
		} finally {
			in = closeStream(in);
		}

		// Now lets see if we have one in the users home directory. This is
		// named .magic.mime as opposed to magic.mime
		try {
			File f = new File(System.getProperty("user.home") + File.separator
					+ ".magic.mime");
			if (f.exists()) {
				in = new FileInputStream(f);
				if (in != null) {
					try {
						parse(f.getAbsolutePath(), new InputStreamReader(in));
					} catch(Exception ex) {
						log.error("Failed to parse .magic.mime file from the users home directory. File will be ignored.", ex);
					}
				}
			}
		}catch(Exception e) {
			log.error("Problem while processing .magic.mime file from the users home directory. File will be ignored.", e);
		} finally {
			in = closeStream(in);
		}

		// Now lets see if we have an environment variable named MAGIC set. This
		// would normally point to a magic or magic.mgc file.
		// As we don't use these file types we will look to see if there is also
		// a magic.mime file at this location for us to use.
		try {
			String name = System.getProperty("MAGIC");
			if (name != null && name.length() != 0) {
				// Strip the .mgc from the end if it's there and add the .mime
				// extension
				if (name.indexOf('.') < 0) {
					name = name + ".mime";
				} else {
					// remove the mgc extension
					name = name.substring(0, name.indexOf('.') - 1) + "mime";
				}
				File f = new File(name);
				if (f.exists()) {
					in = new FileInputStream(f);
					if (in != null) {
						try {
							parse(f.getAbsolutePath(),
									new InputStreamReader(in));
						}catch(Exception ex) {
							log.error("Failed to parse magic.mime file from directory located by environment variable MAGIC. File will be ignored.", ex);
						}
					}
				}
			}
		} catch (Exception e) {
			log.error("Problem while processing magic.mime file from directory located by environment variable MAGIC. File will be ignored.", e);
		} finally {
			in = closeStream(in);
		}

		// Parse the UNIX magic(5) magic.mime files. Since there can be
		// multiple, we have to load all of them.
		// We save, how many entries we have now, in order to fall back to our
		// default magic.mime that we ship,
		// if no entries were read from the OS.

		int mMagicMimeEntriesSizeBeforeReadingOS = mMagicMimeEntries.size();
		Iterator it = magicMimeFileLocations.iterator();
		while (it.hasNext()) {
			parseMagicMimeFileLocation((String) it.next());
		}

		if (mMagicMimeEntriesSizeBeforeReadingOS == mMagicMimeEntries.size()) {
			// Use the magic.mime that we ship
			try {
				String resource = "eu/medsea/mimeutil/magic.mime";
				in = MimeUtil.class.getClassLoader().getResourceAsStream(
						resource);
				if(in != null) {
					try {
						parse("resource:" + resource, new InputStreamReader(in));
					}catch(Exception ex) {
						log.error("Failed to parse internal magic.mime file.", ex);
					}
				}
			} catch (Exception e) {
				log.error("Problem while processing internal magic.mime file.", e);
			} finally {
				in = closeStream(in);
			}
		}
	}

	private static void parseMagicMimeFileLocation(final String location) {
		InputStream is = null;

		List magicMimeFiles = getMagicFilesFromMagicMimeFileLocation(location);

		for (Iterator itFile = magicMimeFiles.iterator(); itFile.hasNext();) {
			File f = (File) itFile.next();
			try {
				if (f.exists()) {
					is = new FileInputStream(f);
					try {
						parse(f.getAbsolutePath(), new InputStreamReader(is));
					}catch(Exception e) {
						log.error("Failed to parse " + f.getName() + ". File will be ignored.");
					}
				}
			} catch (Exception e) {
				log.error(e.getMessage(), e);
			} finally {
				is = closeStream(is);
			}
		}
	}

	private static List getMagicFilesFromMagicMimeFileLocation(
			final String magicMimeFileLocation) {
		List magicMimeFiles = new LinkedList();
		if (magicMimeFileLocation.indexOf('*') < 0) {
			magicMimeFiles.add(new File(magicMimeFileLocation));
		} else {
			int lastSlashPos = magicMimeFileLocation.lastIndexOf('/');
			File dir;
			String fileNameSimplePattern;
			if (lastSlashPos < 0) {
				dir = new File("someProbablyNotExistingFile").getAbsoluteFile()
						.getParentFile();
				fileNameSimplePattern = magicMimeFileLocation;
			} else {
				String dirName = magicMimeFileLocation.substring(0,
						lastSlashPos);
				if (dirName.indexOf('*') >= 0)
					throw new UnsupportedOperationException(
							"The wildcard '*' is not allowed in directory part of the location! Do you want to implement expressions like /path/**/*.mime for recursive search? Please do!");

				dir = new File(dirName);
				fileNameSimplePattern = magicMimeFileLocation
						.substring(lastSlashPos + 1);
			}

			if (!dir.isDirectory())
				return Collections.EMPTY_LIST;

			String s = fileNameSimplePattern.replaceAll("\\.", "\\\\.");
			s = s.replaceAll("\\*", ".*");
			Pattern fileNamePattern = Pattern.compile(s);

			File[] files = dir.listFiles();
			for (int i = 0; i < files.length; i++) {
				File file = files[i];

				if (fileNamePattern.matcher(file.getName()).matches())
					magicMimeFiles.add(file);
			}
		}
		return magicMimeFiles;
	}

	// Parse the magic.mime file
	private static void parse(final String magicFile, final Reader r)
			throws IOException {
		long start = System.currentTimeMillis();

		BufferedReader br = new BufferedReader(r);
		String line;
		ArrayList sequence = new ArrayList();

		long lineNumber = 0;
		line = br.readLine();
		if (line != null)
			++lineNumber;
		while (true) {
			if (line == null) {
				break;
			}
			line = line.trim();
			if (line.length() == 0 || line.charAt(0) == '#') {
				line = br.readLine();
				if (line != null)
					++lineNumber;
				continue;
			}
			sequence.add(line);

			// read the following lines until a line does not begin with '>' or
			// EOF
			while (true) {
				line = br.readLine();
				if (line != null)
					++lineNumber;
				if (line == null) {
					addEntry(magicFile, lineNumber, sequence);
					sequence.clear();
					break;
				}
				line = line.trim();
				if (line.length() == 0 || line.charAt(0) == '#') {
					continue;
				}
				if (line.charAt(0) != '>') {
					addEntry(magicFile, lineNumber, sequence);
					sequence.clear();
					break;
				}
				sequence.add(line);
			}

		}
		if (!sequence.isEmpty()) {
			addEntry(magicFile, lineNumber, sequence);
		}

		if (log.isDebugEnabled())
			log.debug("Parsing \"" + magicFile + "\" took "
					+ (System.currentTimeMillis() - start) + " msec.");
	}

	private static void addEntry(final String magicFile, final long lineNumber,
			final ArrayList aStringArray) {
		try {
			MagicMimeEntry magicEntry = new MagicMimeEntry(aStringArray);
			mMagicMimeEntries.add(magicEntry);
			// Add this to the list of known mime types as well
			if (magicEntry.getMimeType() != null) {
				MimeUtil.addKnownMimeType(magicEntry.getMimeType());
			}
		} catch (InvalidMagicMimeEntryException e) {
			// Continue on but lets print an exception so people can see there
			// is a problem
			log.warn(e.getClass().getName() + ": " + e.getMessage()
					+ ": file \"" + magicFile + "\": before or at line "
					+ lineNumber, e);
		}
	}
}
