tkensdf_reader.cpp

/********************************************************************************
 *   Copyright (c) : Université de Lyon 1, CNRS/IN2P3, UMR5822,                 *
 *                   IP2I, F-69622 Villeurbanne Cedex, France                   *
 *                   Normandie Université, ENSICAEN, UNICAEN, CNRS/IN2P3,       *
 *                   LPC Caen, F-14000 Caen, France                             *
 *   Contibutor(s) :                                                            *
 *      Jérémie Dudouet jeremie.dudouet@cnrs.fr [2020]                          *
 *      Diego Gruyer    diego.gruyer@cnrs.fr    [2020]                          *
 *                                                                              *
 *   Licensed under the MIT License <http://opensource.org/licenses/MIT>.       *
 *   SPDX-License-Identifier: MIT                                               *
 ********************************************************************************/
 
#include <dirent.h>
 
#include <algorithm>
#include <cctype>
#include <cstdint>
#include <iomanip>
#include <sstream>
#include <string>
#include <vector>
 
#include "tkensdf_reader.h"
#include "tklog.h"
 
namespace tkn {
}
 
using namespace tkn;
using namespace std;
 
namespace {
 
bool ascii_equivalent(std::uint32_t codepoint, char &equivalent)
{
   switch (codepoint) {
   case 0x00a0: // no-break space
   case 0x2007: // figure space
   case 0x202f: // narrow no-break space
      equivalent = ' ';
      return true;
   case 0x00b7: // middle dot
   case 0x00d7: // multiplication sign
   case 0x22c5: // dot operator
      equivalent = '*';
      return true;
   case 0x2010: // hyphen
   case 0x2011: // non-breaking hyphen
   case 0x2012: // figure dash
   case 0x2013: // en dash
   case 0x2014: // em dash
   case 0x2015: // horizontal bar
   case 0x2212: // minus sign
      equivalent = '-';
      return true;
   case 0x2018: // left single quotation mark
   case 0x2019: // right single quotation mark
   case 0x2032: // prime
      equivalent = '\'';
      return true;
   case 0x201c: // left double quotation mark
   case 0x201d: // right double quotation mark
   case 0x2033: // double prime
      equivalent = '"';
      return true;
   default:
      return false;
   }
}
 
std::string hex_value(std::uint32_t value, int width)
{
   std::ostringstream output;
   output << std::uppercase << std::hex << std::setfill('0') << std::setw(width) << value;
   return output.str();
}
 
bool is_comment_record(const std::string &record)
{
   if (record.size() < 9) return false;
   const auto field7 = static_cast<char>(std::toupper(static_cast<unsigned char>(record[6])));
   return field7 == 'C' || field7 == 'D' || field7 == 'T';
}
 
bool decode_utf8(const std::string &input, std::size_t &position, std::uint32_t &codepoint)
{
   const auto lead = static_cast<unsigned char>(input[position]);
   std::size_t length = 0;
   std::uint32_t minimum = 0;
 
   if ((lead & 0xe0U) == 0xc0U) {
      length = 2;
      codepoint = lead & 0x1fU;
      minimum = 0x80;
   } else if ((lead & 0xf0U) == 0xe0U) {
      length = 3;
      codepoint = lead & 0x0fU;
      minimum = 0x800;
   } else if ((lead & 0xf8U) == 0xf0U) {
      length = 4;
      codepoint = lead & 0x07U;
      minimum = 0x10000;
   } else {
      return false;
   }
 
   if (position + length > input.size()) return false;
   for (std::size_t offset = 1; offset < length; ++offset) {
      const auto byte = static_cast<unsigned char>(input[position + offset]);
      if ((byte & 0xc0U) != 0x80U) return false;
      codepoint = (codepoint << 6U) | (byte & 0x3fU);
   }
 
   if (codepoint < minimum || codepoint > 0x10ffffU ||
       (codepoint >= 0xd800U && codepoint <= 0xdfffU))
      return false;
 
   position += length;
   return true;
}
 
} // namespace
 
bool tkensdf_reader::normalize_record(tkstring &record, int line_number, bool report)
{
   if (!record.empty() && record.back() == '\r') record.pop_back();
 
   const bool ascii_only = std::none_of(
      record.begin(), record.end(),
      [](const unsigned char byte) { return byte >= 0x80U; });
   if (ascii_only) {
      if (record.size() == tkensdf_record::RSIZE - 1) return true;
      if (report) {
         glog << error_v << "record at line " << line_number << " has " << record.size()
              << " columns instead of " << tkensdf_record::RSIZE - 1 << do_endl;
      }
      record.clear();
      return false;
   }
 
   // Record classification uses columns 6--9. They must remain byte-identical
   // to their logical columns before any UTF-8 substitution is considered.
   for (std::size_t position = 0; position < 9 && position < record.size(); ++position) {
      const auto byte = static_cast<unsigned char>(record[position]);
      if (byte >= 0x80U) {
         if (report) {
            glog << error_v << "non-ASCII byte 0x" << hex_value(byte, 2)
                 << " in the structural ENSDF prefix at line " << line_number
                 << ", column " << position + 1 << do_endl;
         }
         record.clear();
         return false;
      }
   }
 
   const bool comment_record = is_comment_record(record);
 
   std::string normalized;
   normalized.reserve(record.size());
   struct replacement {
      std::uint32_t codepoint;
      std::size_t column;
      char equivalent;
   };
   std::vector<replacement> replacements;
 
   for (std::size_t position = 0; position < record.size();) {
      const auto byte = static_cast<unsigned char>(record[position]);
      if (byte < 0x80U) {
         normalized.push_back(static_cast<char>(byte));
         ++position;
         continue;
      }
 
      std::uint32_t codepoint = 0;
      const auto invalid_position = position;
      const auto column = normalized.size() + 1;
      if (!decode_utf8(record, position, codepoint)) {
         if (report) {
            const auto invalid_byte = static_cast<unsigned char>(record[invalid_position]);
            glog << error_v << "invalid UTF-8 byte 0x" << hex_value(invalid_byte, 2)
                 << " at ENSDF line " << line_number << ", column " << column << do_endl;
         }
         record.clear();
         return false;
      }
 
      if (!comment_record || column < 10) {
         if (report) {
            glog << error_v << "refusing Unicode U+" << hex_value(codepoint, 4)
                 << " outside comment text at ENSDF line " << line_number
                 << ", column " << column << do_endl;
         }
         record.clear();
         return false;
      }
 
      char equivalent = '\0';
      if (!ascii_equivalent(codepoint, equivalent)) {
         if (report) {
            glog << error_v << "unsupported Unicode U+" << hex_value(codepoint, 4)
                 << " at ENSDF line " << line_number << ", column " << column << do_endl;
         }
         record.clear();
         return false;
      }
      normalized.push_back(equivalent);
      replacements.push_back({codepoint, column, equivalent});
   }
 
   if (normalized.size() != tkensdf_record::RSIZE - 1) {
      if (report) {
         glog << error_v << "UTF-8 normalization at ENSDF line " << line_number
              << " produced " << normalized.size() << " columns instead of "
              << tkensdf_record::RSIZE - 1 << do_endl;
      }
      record.clear();
      return false;
   }
 
   if (report) {
      for (const auto &item : replacements) {
         glog << warning_v << "normalized Unicode U+" << hex_value(item.codepoint, 4)
              << " to '" << item.equivalent << "' at ENSDF line " << line_number
              << ", column " << item.column << do_endl;
      }
   }
   record = normalized;
   return true;
}
 
bool tkensdf_reader::open_nuc(const tkstring &_nuc_name, const ensdf_data_type &_ftype)
{
   glog.set_class("ensdf_ascii_reader");
   glog.set_method(tkstring::form("open_nuc(%s,%d)", _nuc_name.data(), _ftype));
 
   tkstring testnuc = _nuc_name.copy().to_lower();
   testnuc.capitalize();
 
   tkstring file_name;
   if (_ftype == kensdf)
      file_name = tkstring::Form("%s/ENSDF/%s.ens", finput_folder.data(), _nuc_name.data());
   else if (_ftype == kxundl)
      file_name = tkstring::Form("%s/XUNDL/%s.ens", finput_folder.data(), _nuc_name.data());
   else {
      glog << warning_v << " database for nucleus " << _nuc_name << " not found" << do_endl;
      glog.clear();
      return false;
   }
 
   fdata_type = _ftype;
   fNucleus = testnuc;
   glog.clear();
 
   return open_file(file_name);
}
 
bool tkensdf_reader::open_file(const tkstring &_file_name)
{
   glog.set_class("ensdf_ascii_reader");
   glog.set_method(tkstring::form("open_file(%s)", _file_name.data()));
 
   if (fEnsdf_file.is_open()) {
      close_file();
   }
   fEnsdf_file.open(_file_name);
 
   tkstring record;
   tkensdf_record the_record;
   tkensdf_ident_rec the_identification_record;
 
   // counters
   int read_record = 0;
 
   // true if a new data set is found
   bool new_dataset = false;
 
   // read the file and create the list of data sets.
   while (true) {
      const std::streampos record_byte_position = fEnsdf_file.tellg();
      getline(fEnsdf_file, record);
 
      if (!fEnsdf_file.good()) break;
 
      if (!normalize_record(record, read_record + 1, true)) {
         read_record++;
         continue;
      }
 
      if (!the_record.set_record(record)) {
         glog << error_v << "error in : " << record << do_endl;
         read_record++;
         continue;
      }
 
      if (!the_record.is_continuation_record()) frecords_counter[the_record.get_record_type()].first++;
      frecords_counter[the_record.get_record_type()].second++;
 
      // end of file
      if (!fEnsdf_file && new_dataset) {
         the_identification_record.fposition.second = read_record;
         fDataSets.push_back(the_identification_record);
         the_identification_record.clear();
         break;
      }
 
      read_record++;
 
      if (the_record.get_record_type() == tkensdf_record::kident && !new_dataset) {
         the_identification_record.set_record(record);
         the_identification_record.set_current_position(read_record);
         the_identification_record.set_start_position(read_record);
         the_identification_record.set_byte_position(record_byte_position);
 
         if (fVerbose) the_identification_record.print(std::cout);
         new_dataset = true;
      }
      if (the_record.get_record_type() == tkensdf_record::kend && new_dataset) {
         the_identification_record.set_stpop_position(read_record);
         fDataSets.push_back(the_identification_record);
         the_identification_record.clear();
         new_dataset = false;
      }
   } // is_open
   if (fVerbose) glog << info << fDataSets.size() << " datasets have been found in the ENSDF file " << _file_name << do_endl;
 
   glog.clear();
 
   return fEnsdf_file.is_open();
}
 
void tkensdf_reader::print_datasets()
{
   if (!fEnsdf_file.is_open()) {
      glog << error << "No ENSDF database opened" << do_endl;
      return;
   }
   tkstring data_type = "ENSDF";
   if (fdata_type == kxundl) data_type = "XUNDL";
   glog << info << "Database type: " << data_type << do_endl;
   glog << info << fDataSets.size() << " datasets have been found for nucleus " << fNucleus << ":" << do_endl;
   for (auto &key : fDataSets) key.print(std::cout);
}
 
bool tkensdf_reader::first_record(tkensdf_ident_rec *_dataset, tkstring &record)
{
   // to remove any error flag
   fEnsdf_file.clear();
 
   if (_dataset == nullptr) return false;
   _dataset->set_current_position(_dataset->get_start_position());
   fEnsdf_file.seekg(_dataset->get_byte_position());
   getline(fEnsdf_file, record);
   normalize_record(record, _dataset->get_current_position(), false);
   return fEnsdf_file.good();
}
 
bool tkensdf_reader::next_record(tkensdf_ident_rec *_dataset, tkstring &record)
{
   if (_dataset == nullptr) return false;
   _dataset->set_current_position(_dataset->get_current_position() + 1);
   if (_dataset->get_current_position() >= _dataset->get_stop_position()) return false;
   getline(fEnsdf_file, record);
   normalize_record(record, _dataset->get_current_position(), false);
   return fEnsdf_file.good();
}
 
void tkensdf_reader::close_file()
{
   fDataSets.clear();
   fEnsdf_file.close();
   fEnsdf_file.clear();
   fNucleus.clear();
}
 
void tkensdf_reader::print_record_counters()
{
   std::cout << std::endl;
   glog << info << "Record counters:" << do_endl;
 
   size_t maxsize1 = 0;
   size_t maxsize2 = 0;
   for (auto i : frecords_counter) {
      if (tkstring::Form("%lld", i.second.first).length() > maxsize1) maxsize1 = tkstring::Form("%lld", i.second.first).length();
      if (tkstring::Form("%lld", i.second.second).length() > maxsize2) maxsize2 = tkstring::Form("%lld", i.second.second).length();
   }
   std::cout << left << " - Record Type     :  counts without(with) continuation records" << std::endl;
   for (auto i : frecords_counter) {
      if (i.first == tkensdf_record::kident)
         std::cout << left << " - identification  : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::khistory)
         std::cout << left << " - History         : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kq_value)
         std::cout << left << " - Q-value         : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kxref)
         std::cout << left << " - X-ref           : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kcomment)
         std::cout << left << " - Comment         : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kparent)
         std::cout << left << " - Parent          : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::knorm)
         std::cout << left << " - Normalisation   : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kprodnorm)
         std::cout << left << " - Production norm : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::klevel)
         std::cout << left << " - Level           : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kbeta)
         std::cout << left << " - Beta            : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kec)
         std::cout << left << " - EC              : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kalpha)
         std::cout << left << " - Alpha           : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kparticle)
         std::cout << left << " - Particle        : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kgamma)
         std::cout << left << " - Gamma           : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kreference)
         std::cout << left << " - Reference       : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kend)
         std::cout << left << " - End             : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
      else if (i.first == tkensdf_record::kunknown)
         std::cout << left << " - Unknown         : " << setw(maxsize1 + 1) << i.second.first << " (" << setw(maxsize2) << i.second.second << ")" << std::endl;
   }
   std::cout << std::endl;
}