// llil2vec.cpp.
// Vector version of llil2grt.pl.
// g++ compile on Linux:
//    g++ -o llil2vec -std=c++11 -Wall -O3 llil2vec.cpp
// This g++ command also works with mingw C++ compiler (https://source
+forge.net/projects/mingw-w64)
// that comes bundled with Strawberry Perl (C:\Strawberry\c\bin\g++.ex
+e).
// Example run: llil2vec big1.txt big2.txt big3.txt >vec.tmp

#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <cstdio>

#include <string>
#include <array>
#include <vector>
#include <set>

#include <algorithm>
#include <utility>
#include <iterator>

#include <iostream>
#include <fstream>
#include <sstream>

static_assert(sizeof(size_t) == sizeof(int64_t), "size_t too small, ne
+ed a 64-bit compile");

// -------------------------------------------------------------------
+---------

// Crude hack to see Windows Private Bytes in Task Manager by sleeping
+ at
// program end (see also sleep hack at end of main)
//    #include <chrono>
//    #include <thread>

// -------------------------------------------------------------------
+---------

typedef long long llil_int_type;

// Note: all words in big1.txt, big2.txt, big3.txt are <= 6 chars in l
+ength
// To use (limited length) fixed length strings uncomment the next lin
+e
// #define MAX_STR_LEN_L 6

#ifdef MAX_STR_LEN_L
using str_arr_type     = std::array<char, MAX_STR_LEN_L + 1>;    // +1
+ for trailing '\0'
using str_int_type     = std::pair<str_arr_type, llil_int_type>;
using int_str_type     = std::pair<llil_int_type, str_arr_type>;
#else
using str_int_type     = std::pair<std::string, llil_int_type>;
using int_str_type     = std::pair<llil_int_type, std::string>;
#endif

using vec_str_int_type = std::vector<str_int_type>;
using vec_int_str_type = std::vector<int_str_type>;
using set_int_str_type = std::set<int_str_type>;

// Mimic the Perl get_properties subroutine --------------------------
+--

// Limit line length and use ANSI C functions to try to boost performa
+nce
#define MAX_LINE_LEN_L 255

static void get_properties(
   int                nfiles,      //  in: the number of input files
   char*              fname[],     //  in: the input file names
   vec_int_str_type&  vec_ret)     // out: a vector of properties
{
   FILE* fh;
   char line[MAX_LINE_LEN_L+1];
   char* word;
   llil_int_type count;

   for (int i = 0; i < nfiles; ++i) {
      fh = ::fopen(fname[i], "r");
      if (fh == NULL) {
         std::cerr << "Error opening '" << fname[i] << "' : errno=" <<
+ errno << "\n";
         continue;
      }
      while ( ::fgets(line, MAX_LINE_LEN_L, fh) != NULL ) {
         word  = ::strtok(line, "\t");
         count = ::atoll( ::strtok(NULL, "\n") );
#ifdef MAX_STR_LEN_L
         str_arr_type fixword { { '\0', '\0', '\0', '\0', '\0', '\0', 
+'\0' } };
         ::strcpy( fixword.data(), word );
         vec_ret.emplace_back( -count, fixword );
#else
         vec_ret.emplace_back( -count, word );
#endif
      }
      ::fclose(fh);
   }

   // Needs to be sorted by word for later sum of adjacent count field
+s to work
   std::sort( vec_ret.begin(), vec_ret.end(),
      [](const int_str_type& left, const int_str_type& right) { return
+ left.second < right.second; }
   );
}

// -------------------------------------------------------------------
+--

int main(int argc, char* argv[])
{
   if (argc < 2) {
      std::cerr << "usage: llil2vec file1 file2 ... >out.txt\n";
      return 1;
   }

#ifdef MAX_STR_LEN_L
   std::cerr << "llil2vec (fixed string length=" << MAX_STR_LEN_L << "
+) start\n";
#else
   std::cerr << "llil2vec start\n";
#endif
   time_t  tstart1 = ::time(NULL);
   clock_t cstart1 = ::clock();

   // Create the vector of properties
   vec_int_str_type vec_ret;
   get_properties(argc - 1, &argv[1], vec_ret);

   clock_t cend1  = ::clock();
   double ctaken1 = (double) (cend1 - cstart1) / (double)CLOCKS_PER_SE
+C;
   std::cerr << "get_properties      CPU time : " << ctaken1 << " secs
+\n";

   clock_t cstart2 = ::clock();

   // To avoid calling sort(), create an inverted std::set container
   // Note: negative count gives desired ordering
   set_int_str_type myset;
   auto it = vec_ret.begin();
   int_str_type  kv_last = *it;
   llil_int_type count   = it->first;
   for (++it; it != vec_ret.end(); ++it) {
      if ( it->second == kv_last.second ) {
         count += it->first;
      }
      else {
         myset.emplace_hint( myset.end(), count, kv_last.second );
         kv_last = *it;
         count   = it->first;
      }
   }
   myset.emplace_hint( myset.end(), count, kv_last.second );

   clock_t cend2s = ::clock();

   // Output the (already sorted) std::set - no sort() function requir
+ed
   // Note: fix up negative count via -n.first
#ifdef MAX_STR_LEN_L
   for ( auto const& n : myset ) std::cout << n.second.data() << '\t' 
+<< -n.first << '\n';
#else
   for ( auto const& n : myset ) std::cout << n.second        << '\t' 
+<< -n.first << '\n';
#endif

   clock_t cend2 = ::clock();
   time_t  tend2 = ::time(NULL);

   long    ttaken   = static_cast<long>(::difftime(tend2, tstart1) + 0
+.5);
   double  ctaken   = (double) (cend2 - cstart1) / (double)CLOCKS_PER_
+SEC;

   double  ctaken2s = (double) (cend2s - cstart2) / (double)CLOCKS_PER
+_SEC;
   double  ctaken2o = (double) (cend2  - cend2s)  / (double)CLOCKS_PER
+_SEC;

   std::cerr << "emplace set sort    CPU time : " << ctaken2s << " sec
+s\n";
   std::cerr << "write stdout        CPU time : " << ctaken2o << " sec
+s\n";

   std::cerr << "total               CPU time : " << ctaken   << " sec
+s\n";
   std::cerr << "total        wall clock time : " << ttaken   << " sec
+s\n";

   // Hack to see Private Bytes in Windows Task Manager (uncomment nex
+t line so process doesn't exit too quickly)
   //   std::this_thread::sleep_for(std::chrono::milliseconds(90000000
+));

   return 0;
}
[download]