custom_compression_generate_model.mnh

bigrams(line:String)->begin
  B:=line.bytes.ord;
  B.size<2 ? void : [B.leading,B.tail].transpose;
end;

datastore BIGRAM_COUNTS:=[].toMap;

updateBigramCounts->BIGRAM_COUNTS:=
  allFiles('C:\dev\private',['*.txt','*.mnh'])
  .pMap((file)->note('Reading: ',file) orElse
                file.fileLines
                    .each(line,line.bigrams,|)
                    .elementFrequency)
  .agg((x,y)->mergeMaps(x,y,::+))
  .filter((key,count)->count>1)
  .toMap;

Base:=map([0..255],(x)->[x,1]).toMap;

memoized defaultFrequency->BIGRAM_COUNTS.map((key,value)->key[0]=>value).{$L.getInner(1).group($L.getInner(0),::+)};

memoized frequencyByPredecessor->
BIGRAM_COUNTS.map((key,value)->key|value).group(0)
.map((key,value)->key=>group(value.getInner(2),value.getInner(1),(x,y)->fail(x,'-',y)))
.toMap;

tabulate(freqMap:Map)-> freqMap.mergeMaps(Base,::orElse).sort.getInner(1).join(",");

main('update')->updateBigramCounts;

main->begin
  printf("CONST initialState=#{[0..255].minus(frequencyByPredecessor.getInner(0)).min};");
  print('PROCEDURE ensureModel;');
  printf('  CONST F_D: T_freqTable=(%s);',tabulate(defaultFrequency));
  frequencyByPredecessor.sort.map((key,value)->printf('        F_{key}: T_freqTable=({tabulate(value)});'));
  print('  begin');
  initialState:=void;
  [0..255].each(key,begin
    sourceData:=frequencyByPredecessor[key].isVoid
                ? begin initialState:=initialState orElse key; 'F_D'; end
                : f'F_{key}';
    printf("    buildCode({sourceData}, decode_tree[#{key}], encode_table[#{key}]);");
  end);
  assert(!isVoid(initialState));
  print('  end;');
end;