build.gradle

import java.text.Collator

plugins {
    id 'java'
    id 'groovy'
    id 'eclipse'
}

compileJava.options.encoding = 'UTF-8'

version=new File("${projectDir}/VERSION").text.trim()
group = 'org.dict_uk'

String langCode="uk"

String affixDir="data/affix"
String dataDir="data/dict"

String outputDir="out"
String prevDir="../../out/prev"


repositories {
    mavenCentral()
}

//configurations {
//    groovyScript
//}

dependencies {
    implementation group: 'org.apache.groovy', name: 'groovy', version: '4.0.+'
    implementation group: 'org.apache.groovy', name: 'groovy-swing', version: '4.0.+'
    implementation group: 'org.apache.groovy', name: 'groovy-cli-picocli', version: '4.0.+'
    implementation 'info.picocli:picocli:4.6.1'

    implementation group: 'commons-cli', name: 'commons-cli', version: '1.4'
    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.+'

    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.+'
    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.+'

//    groovyScript localGroovy()
}
/*
java {
    toolchain {
        languageVersion = JavaLanguageVersion.of(11)
    }
}
*/

//compileGroovy {
//    exclude '**/tools/other/*.groovy'
//}

eclipse {
  classpath {
    defaultOutputDir = file('build')
        file.whenMerged {
            entries.each { entry ->
                if (entry.kind == 'src' && entry.hasProperty('output')) {
                    entry.output = entry.output ? entry.output.replace('bin/', "build_/")
                                    : entry.path.replaceFirst(/src\/(main\/).*/, 'build_/$1')
//                    logger.lifecycle ":: $entry -- ${entry.output}"
                }
            }
        }
  }
}


//
// Auxilliary targets
//

task reverseVerbs (type: JavaExec, dependsOn: classes) {
    def inputFile = "$affixDir/v.aff"
    inputs.file inputFile
    outputs.file "$affixDir/vr.aff"

    classpath = sourceSets.main.runtimeClasspath
    mainClass = "org.dict_uk.expand.VerbReverse"
    args file(inputFile), file(outputs.files[0])

//    standardInput = file(inputFile).newInputStream()
//    standardOutput = file(outputs.files[0]).newOutputStream()
}

task reverseAdvp (type: JavaExec, dependsOn: classes) {
    def inputFile = "$affixDir/v_advp.aff"
    inputs.file inputFile
    outputs.file "$affixDir/vr_advp.aff"

    classpath = sourceSets.main.runtimeClasspath
    mainClass = "org.dict_uk.expand.VerbReverse"

    args file(inputFile), file(outputs.files[0])
}



task autogen (type: JavaExec, dependsOn: classes) {
	classpath = sourceSets.main.runtimeClasspath
	mainClass = "org.dict_uk.tools.Autogen"

	String inputFile1 = "$dataDir/base.lst"
    String inputFile2 = "$dataDir/base-compound.lst"
	String inputFile3 = "$dataDir/base-compound_1992.lst"
	
    inputs.file inputFile1
    inputs.file inputFile2
    inputs.file inputFile3
    outputs.file "$dataDir/base-autogen.lst"

	args inputFile1, inputFile2, inputFile3, "$dataDir/base-autogen.lst"
}

task reverseAffix (dependsOn: [reverseVerbs, reverseAdvp]) {
}


task sortDict(type: JavaExec) {
	classpath = sourceSets.main.runtimeClasspath
	mainClass = "org.dict_uk.tools.Sort"

	workingDir = file("data/dict")
	
	def files = "alt.lst base-abbr.lst base.lst base-compound.lst colors.lst dot-abbr.lst"
	files += " geo-ukr-hydro.lst geo-other.lst"  //  geo-ukr-koatuu.lst
	files += " slang.lst twisters.lst invalid.lst invalid-compound.lst names-anim.lst names-other.lst arch.lst base-compound_1992.lst"
	files += " names-composite.lst base-composite.lst subst.lst vulgar.lst"

	args files.split()
}


task dups {
  doLast {
	def dictDir = file("data/dict")
	
	def files = dictDir.listFiles().findAll { it.name.endsWith('.lst') \
	    && ! (it.name in ['dot-abbr.lst', 'geo-ukr-hydro.lst', 'geo-ukr-koatuu.lst']) }
	
	def lines = files.collect { File f ->
            boolean geo = f.name.contains('geo')
            f.readLines()
            .findAll { ! it.startsWith('#') && ! it.startsWith(' +cs=') } 
            .collect { it
//            .replaceFirst(/^ \+cs=/, '')
                .trim()
                .replaceAll(/\s*#.*/, '')
                .replaceAll(/( \/[a-z0-9]+)(\s+|$)/, '$1... ')
                .replaceAll(/\.[^<.: ]+/, '...')
                .replaceAll(/\s*:coll/, '')
                .replaceAll(/:&adjp:(actv|pasv)(:(imperf|perf))+/, '') 
            }
            .collect { line ->
                geo ? "$line :geo" : line 
            }
        }
        .flatten()
	    .grep { it }

//    lines = lines.grep { ! it.startsWith('#') }

    def dups = lines.countBy{it}.grep{it.value > 1 }.collect{it.key}

    java.text.Collator coll = java.text.Collator.getInstance(new Locale("uk", "UA"));
    coll.setStrength(java.text.Collator.IDENTICAL)
    coll.setDecomposition(java.text.Collator.NO_DECOMPOSITION)


    println dups.toSorted(coll).join("\n")
    println "Found ${dups.size} dups"
  }
}

task xps {
    doLast {
      def xps = [:].withDefault{ [] }
      def dictDir = file("data/dict")
      
      def files = dictDir.listFiles().findAll { it.name.endsWith('.lst') }
      
      def lines = files.collect {  it.readLines()  }.flatten() \
          .collect { it.replaceFirst(/#.*/, '') }
          .findAll{ it.contains(":xp") && ! it.startsWith("+cs") && ! it.startsWith('#') }
          .collect { it
              it.replaceAll(/^([^ ]+)\h.*?(:xp.).*/, '$1 $2')
          }
  
      lines.each{ 
          def (base, xp) = it.split(/ /)
          xps[base] << xp 
      }

      xps.each { k,v -> v.sort() }

      xps.each { k,v -> 
          if( v[0] != ':xp1' ) println "out of order: $k: $v"
          else if( v.size() == 1 ) println "single: $k: $v"
          else {
              def dups = v.countBy{it}.grep{it.value > 1}.collect{it.key}
              if( dups ) println "dups: $k: $dups"
          } 
      }
      
//      java.text.Collator coll = java.text.Collator.getInstance(new Locale("uk", "UA"));
//      coll.setStrength(java.text.Collator.IDENTICAL)
//      coll.setDecomposition(java.text.Collator.NO_DECOMPOSITION)
  
  }
}

task checkReplacements {
  doLast {
	def dictDir = file("data/dict")
	
	def files = dictDir.listFiles().findAll { it.name.endsWith('.lst') || it.name.endsWith('replace.txt') }
	
	def replWords = files.collect { it.text.split("\n")  } \
	    .flatten() \
	    .findAll { it.contains(" #> ") }
	    .collect { 
			def repls = it.split(" #> ")[1].trim().split(/[;, \|]+/)
			def word = it.split(' ', 2)[0]
			if( word in repls ) {
				println "Replacement is same as word: $word"
			} 
			repls
		}
	    .flatten()
	    .collect { it.replaceAll(/[()]/, '').replaceFirst(/-таки$/, '') }
	    .findAll { it =~ /[а-яіїєґ]/ }
	    .unique().sort()

    def spellWords = new File("out/words_spell.txt").text.split('\n')
    spellWords += new File("data/dict/slang.lst").text.split('\n').collect{ it.replaceFirst(/ .*/, '') }
	spellWords += new File("data/dict/arch.lst").text.split('\n').collect{ it.replaceFirst(/ .*/, '') }
	
    println "Unique replacement words: ${replWords.size}"
    
    replWords.removeAll(spellWords)


    println "Unknown:\n" + replWords.join("\n")
    println "Total uknown: ${replWords.size}"
  }
}

task checkSemtags(type: JavaExec){ 
    classpath = sourceSets.test.runtimeClasspath
    mainClass = "org.dict_uk.check.CheckSemtags"

    workingDir = projectDir

//    def opts = " --indent --mfl --stats --wordlist"
//    args "--aff", "../data/affix"
//    args "--dict", "dict"
//    args opts.split()
  }
  

task showExpandCommand {
  doLast {
    def opts = " --indent --mfl --stats --wordlist --time"
    opts += " -aff data/affix"
    opts += " -dict data/dict"

    def cmd = "groovy -cp "
    cmd += sourceSets.main.output.classesDir
    cmd += ":" + sourceSets.main.output.resourcesDir
    cmd += ":" + configurations.runtime.asFileTree.files.join(':')
//    cmd += " org.dict_uk.expand.ExpandAll"
    cmd += " src/main/groovy/org/dict_uk/expand/ExpandAll.groovy"
    cmd += opts
    
    println cmd
  }
}

//
// Testing targets
//

def getAdjustedText(filename) {
       new File(filename).text.replace("\r\n", "\n")
}


task testExpand(type: JavaExec, dependsOn: [reverseAffix, autogen, classes]) {
    classpath = sourceSets.main.runtimeClasspath
    mainClass = "org.dict_uk.expand.ExpandAll"

    workingDir = file('test')

    def opts = " --indent --mfl --stats --wordlist"
    args "--aff", "../data/affix"
    args "--dict", "dict"
    args opts.split()

//    jvmArgs "-XX:+FlightRecorder -XX:StartFlightRecording=disk=true,dumponexit=true,filename=recording.jfr".split()

    doLast {
        def file="dict_corp_lt.txt"
        def noTestOutputChange = getAdjustedText("$workingDir/$file") == getAdjustedText("$workingDir/prev/$file")
        assert noTestOutputChange

        file="dict_corp_vis.txt"
        def noTestVisOutputChange = getAdjustedText("$workingDir/$file") == getAdjustedText("$workingDir/prev/$file")
        assert noTestVisOutputChange

//        def noTestTagsOutputChange = getAdjustedText("$workingDir/tags.txt") == getAdjustedText("$workingDir/prev/tags.txt")
//        assert noTestTagsOutputChange
    }
}


//
// Main targets
//

task expandForCorp(type: JavaExec, dependsOn: [reverseAffix, autogen/*, testExpand*/, dups]) {
    classpath = sourceSets.main.runtimeClasspath
    mainClass = "org.dict_uk.expand.ExpandAll"

    workingDir = file(outputDir)

    def opts = "--indent --mfl --stats --wordlist --time"
    // opts += "--removeWithRegex verb.*:inf:short"            // remove colloquial short verb forms

    args "--aff", "../data/affix"
    args "--dict", "../data/dict"
    args opts.split()

    jvmArgs "-Xms4000m -Xmx5200m".split()
    // JDK8
//    jvmArgs "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=dumponexit.jfr".split()
    // JDK11
//    jvmArgs "-XX:+FlightRecorder -XX:StartFlightRecording=disk=true,dumponexit=true,filename=recording.jfr".split()
//    jvmArgs "-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n"
}

def get_diff_cmd(outputDir, file) {
    return "diff -u $outputDir/prev/$file $outputDir/$file > $outputDir/${file}.diff || echo 'File $file differs!'"
}


task diff(type: Exec) {
    def dictFile="dict_corp_lt.txt"

//    def cmd = get_diff_cmd(outputDir, dictFile)
//    cmd += "; " + get_diff_cmd(outputDir, 'words.txt')
//    cmd += "; " + get_diff_cmd(outputDir, 'words_spell.txt')
//    cmd += "; " + get_diff_cmd(outputDir, 'lemmas.txt')
//    cmd += "; " + get_diff_cmd(outputDir, 'tags.txt')

    dictFile="dict_corp_vis.txt"
    def cmd = get_diff_cmd(outputDir, dictFile)

    commandLine "sh", "-c", "${cmd}"
}

task expand(dependsOn: expandForCorp) {
}


task expandInteractive(type: JavaExec, dependsOn: reverseAffix) {
    classpath = sourceSets.main.runtimeClasspath
    mainClass = "org.dict_uk.expand.Expand"

    workingDir = file(outputDir)
    standardInput = System.in

    def opts = "-f"
    if( ! project.hasProperty('flat') ) {
        opts += " --indent"
    }
    args "--aff", "../data/affix"
    args "--dict", "-"
    args opts.split()
}



task runTool(type: JavaExec, dependsOn: compileGroovy) {
    classpath = sourceSets.main.output + configurations.runtimeClasspath
    mainClass = "groovy.ui.GroovyMain"
    args "src/tools/groovy/" + (project.hasProperty('tool') ? project.tool : '')
    if( project.hasProperty('file') ) {
        println "Reading word list from: " + project.file
        args project.file
    }
}


//
// deploy LT dict
//

task deployLtDict(type: GradleBuild) {
    dir = 'distr/morfologik-ukrainian'
    tasks = ['deployLtDict']
}


// windows have non-unicode encoding set by default
String osName = System.getProperty("os.name").toLowerCase();
if ( osName.contains("windows")) {
    if( ! "UTF-8".equals(System.getProperty("file.encoding"))
            || ! "UTF-8".equals(java.nio.charset.Charset.defaultCharset()) ) {
        System.setOut(new PrintStream(System.out,true,"UTF-8"))

        println "On Windows to get unicode handled correctly you need to set environment variable before running expand:"
        println "\tbash:"
        println "\t\texport JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF-8"
        println "\tcmd:"
        println "\t\t(change Font to 'Lucida Console' in cmd window properties)"
        println "\t\tchcp 65001"
        println "\t\tset JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF-8"
        println "\n\tNOTE: bash shell (e.g. git bash) is recommended: only in bash the output will handle all cyrillics and expandInteractive only supported in bash"

        // poor man's safety net (may work but still will not show all cyrillic characters)
        tasks.each { task ->
            if( task instanceof JavaExec ) {
                task.jvmArgs '-Dfile.encoding=UTF-8'
            }
        }
    }
}

task fatJar(type: Jar) {
    manifest {
        attributes 'Implementation-Title': 'dict_uk classes',
            'Implementation-Version': archiveVersion
    }
    archiveBaseName = project.name + '-all'
    from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
    with jar
}