package org.apache.lucene.benchmark.utils;
/**
 * Copyright 2005 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


/**
 * Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline, Body
 */
public class ExtractReuters
{
    private File reutersDir;
    private File outputDir;
    private static final String LINE_SEPARATOR = System.getProperty("line.separator");

    public ExtractReuters(File reutersDir, File outputDir)
    {
        this.reutersDir = reutersDir;
        this.outputDir = outputDir;
        System.out.println("Deleting all files in " + outputDir);
        File [] files = outputDir.listFiles();
        for (int i = 0; i < files.length; i++)
        {
            files[i].delete();
        }

    }

    public void extract()
    {
        File [] sgmFiles = reutersDir.listFiles(new FileFilter()
        {
            public boolean accept(File file)
            {
                return file.getName().endsWith(".sgm");
            }
        });
        if (sgmFiles != null && sgmFiles.length > 0)
        {
            for (int i = 0; i < sgmFiles.length; i++)
            {
                File sgmFile = sgmFiles[i];
                extractFile(sgmFile);
            }
        }
        else
        {
            System.err.println("No .sgm files in " + reutersDir);
        }
    }

    Pattern EXTRACTION_PATTERN = Pattern.compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");

    private static String[] META_CHARS
            = {"&", "<", ">", "\"", "'"};

    private static String[] META_CHARS_SERIALIZATIONS
            = {"&", "<", ">", """, "'"};

    /**
     * Override if you wish to change what is extracted
     *
     * @param sgmFile
     */
    protected void extractFile(File sgmFile)
    {
        try
        {
            BufferedReader reader = new BufferedReader(new FileReader(sgmFile));

            StringBuffer buffer = new StringBuffer(1024);
            StringBuffer outBuffer = new StringBuffer(1024);

            String line = null;
            int docNumber = 0;
            while ((line = reader.readLine()) != null)
            {
                //when we see a closing reuters tag, flush the file

                if (line.indexOf("</REUTERS") == -1) {
                    //Replace the SGM escape sequences

                    buffer.append(line).append(' ');//accumulate the strings for now, then apply regular expression to get the pieces,
                }
                else
                {
                    //Extract the relevant pieces and write to a file in the output dir
                    Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
                    while (matcher.find())
                    {
                        for (int i = 1; i <= matcher.groupCount(); i++)
                        {
                            if (matcher.group(i) != null)
                            {
                                outBuffer.append(matcher.group(i));
                            }
                        }
                        outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
                    }
                    String out = outBuffer.toString();
                    for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++)
                    {
                        out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
                    }
                    File outFile = new File(outputDir, sgmFile.getName() + "-" + (docNumber++) + ".txt");
                    //System.out.println("Writing " + outFile);
                    FileWriter writer = new FileWriter(outFile);
                    writer.write(out);
                    writer.close();
                    outBuffer.setLength(0);
                    buffer.setLength(0);
                }
            }
            reader.close();
        }

        catch (
                IOException e
                )

        {
            throw new RuntimeException(e);
        }
    }


    public static void main(String[] args)
    {
        if (args.length != 2)
        {
            printUsage();
        }
        File reutersDir = new File(args[0]);

        if (reutersDir.exists())
        {
            File outputDir = new File(args[1]);
            outputDir.mkdirs();
            ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
            extractor.extract();
        }
        else
        {
            printUsage();
        }
    }

    private static void printUsage()
    {
        System.err.println("Usage: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
    }
}