// CGNS Assumptions:
// * All boundary conditions are listed as Family nodes at the "top" level.
// * Single Base.
// * ZoneGridConnectivity is 1to1 with point lists for unstructured

// Copyright(C) 2015, 2016
// Sandia Corporation. Under the terms of Contract
// DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
// certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Sandia Corporation nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <Ioss_CodeTypes.h>
#include <Ioss_Utils.h>
#include <cassert>
#include <cgns/Iocgns_ParallelDatabaseIO.h>
#include <cgns/Iocgns_Utils.h>
#include <fstream>
#include <iostream>
#include <mpi.h>
#include <numeric>
#include <stddef.h>
#include <string>
#include <sys/select.h>
#include <time.h>
#include <vector>

#include <cgnsconfig.h>
#include <pcgnslib.h>

#if !defined(CGNSLIB_H)
#error "Could not include cgnslib.h"
#endif

#include "Ioss_CommSet.h"
#include "Ioss_DBUsage.h"
#include "Ioss_DatabaseIO.h"
#include "Ioss_ElementBlock.h"
#include "Ioss_ElementTopology.h"
#include "Ioss_EntityType.h"
#include "Ioss_NodeBlock.h"
#include "Ioss_SideBlock.h"
#include "Ioss_SideSet.h"
#include "Ioss_StructuredBlock.h"
#include "Ioss_TerminalColor.h"

#include "Ioss_Field.h"
#include "Ioss_IOFactory.h"
#include "Ioss_ParallelUtils.h"
#include "Ioss_Property.h"
#include "Ioss_Region.h"
#include "Ioss_State.h"
#include "Ioss_Utils.h"
#include "Ioss_VariableType.h"

namespace {
}

namespace Iocgns {

  ParallelDatabaseIO::ParallelDatabaseIO(Ioss::Region *region, const std::string &filename,
                                         Ioss::DatabaseUsage db_usage, MPI_Comm communicator,
                                         const Ioss::PropertyManager &props)
      : Ioss::DatabaseIO(region, filename, db_usage, communicator, props), cgnsFilePtr(-1),
        nodeCount(0), elementCount(0), m_zoneType(CG_ZoneTypeNull)
  {
    usingParallelIO = true;
    dbState         = Ioss::STATE_UNKNOWN;

#ifdef IOSS_DEBUG_OUTPUT
    if (myProcessor == 0) {
      std::cout << "CGNS ParallelDatabaseIO using " << CG_SIZEOF_SIZE << "-bit integers.\n"
                << "                        using the parallel CGNS library and API.\n";
    }
#endif
    if (CG_SIZEOF_SIZE == 64) {
      set_int_byte_size_api(Ioss::USE_INT64_API);
    }

    openDatabase();
  }

  ParallelDatabaseIO::~ParallelDatabaseIO()
  {
    for (auto &gtb : m_globalToBlockLocalNodeMap) {
      delete gtb.second;
    }
    cgp_close(cgnsFilePtr);
  }

  void ParallelDatabaseIO::openDatabase__() const
  {
    if (cgnsFilePtr < 0) {
      int mode = is_input() ? CG_MODE_READ : CG_MODE_WRITE;
#if 0
      // Currently, cgp_mpi_comm returns an internal NO_ERROR value which
      // is equal to -1.  There is an issue submitted for this.
      CGCHECK(cgp_mpi_comm(util().communicator()));
#else
      cgp_mpi_comm(util().communicator());
#endif
      CGCHECK(cgp_pio_mode(CGP_COLLECTIVE));
      int ierr = cgp_open(get_filename().c_str(), mode, &cgnsFilePtr);
      if (ierr != CG_OK) {
        // NOTE: Code will not continue past this call...
        std::ostringstream errmsg;
        errmsg << "ERROR: Problem opening file '" << get_filename() << "' for "
               << (is_input() ? "read" : "write") << " access. "
               << "CGNS Error: '" << cg_get_error() << "'";
        IOSS_ERROR(errmsg);
      }
#if 0
      // This isn't currently working since CGNS currently has chunking
      // disabled for HDF5 files and compression requires chunking.
      if (!is_input()) {
        if (properties.exists("COMPRESSION_LEVEL")) {
          int comp = properties.get("COMPRESSION_LEVEL").get_int();
          cg_configure(CG_CONFIG_HDF5_COMPRESS, (void*)comp);
        }
      }
#endif
    }
    assert(cgnsFilePtr >= 0);
  }

  void ParallelDatabaseIO::closeDatabase__() const
  {
    if (cgnsFilePtr != -1) {
      CGCHECK(cgp_close(cgnsFilePtr));
    }
    cgnsFilePtr = -1;
  }

  void ParallelDatabaseIO::finalize_database()
  {
    if (is_input()) {
      return;
    }

    if (m_timesteps.empty()) {
      return;
    }

    Utils::finalize_database(cgnsFilePtr, m_timesteps, get_region(), myProcessor);
  }

  void ParallelDatabaseIO::release_memory__()
  {
    nodeMap.release_memory();
    elemMap.release_memory();
    try {
      decomp.reset();
    }
    catch (...) {
    }
  }

  int64_t ParallelDatabaseIO::node_global_to_local__(int64_t global, bool must_exist) const
  {
    // TODO: Fix
    return global;
  }

  int64_t ParallelDatabaseIO::element_global_to_local__(int64_t global) const
  {
    // TODO: Fix
    return global;
  }

  void ParallelDatabaseIO::read_meta_data__()
  {
    openDatabase();

    // Determine the number of bases in the grid.
    // Currently only handle 1.
    cgsize_t n_bases = 0;
    CGCHECK(cg_nbases(cgnsFilePtr, &n_bases));
    if (n_bases != 1) {
      std::ostringstream errmsg;
      errmsg << "CGNS: Too many bases; only support files with a single bases at this time";
      IOSS_ERROR(errmsg);
    }

    get_step_times__();

    m_zoneType = Utils::check_zone_type(cgnsFilePtr);

    // In CGNS, there are duplicated nodes at block boundaries.
    // We typically only want to retain one copy of these and ignore the other.
    properties.add(Ioss::Property("RETAIN_FREE_NODES", "NO"));

    if (int_byte_size_api() == 8) {
      decomp = std::unique_ptr<DecompositionDataBase>(
          new DecompositionData<int64_t>(properties, util().communicator()));
    }
    else {
      decomp = std::unique_ptr<DecompositionDataBase>(
          new DecompositionData<int>(properties, util().communicator()));
    }
    assert(decomp != nullptr);
    decomp->decompose_model(cgnsFilePtr, m_zoneType);

    if (m_zoneType == CG_Structured) {
      handle_structured_blocks();
    }
    else if (m_zoneType == CG_Unstructured) {
      handle_unstructured_blocks();
    }

    Utils::add_transient_variables(cgnsFilePtr, m_timesteps, get_region(), myProcessor);
  }

  void ParallelDatabaseIO::handle_unstructured_blocks()
  {
    get_region()->property_add(
        Ioss::Property("global_node_count", (int64_t)decomp->global_node_count()));
    get_region()->property_add(
        Ioss::Property("global_element_count", (int64_t)decomp->global_elem_count()));

    nodeCount    = decomp->ioss_node_count();
    elementCount = decomp->ioss_elem_count();

    // ========================================================================
    // Get the number of families in the mesh...
    // Will treat these as sidesets if they are of the type "FamilyBC_t"
    Utils::add_sidesets(cgnsFilePtr, this);

    // ========================================================================
    // Get the number of zones (element blocks) in the mesh...
    int base = 1;
    int i    = 0;
    for (auto &block : decomp->m_elementBlocks) {
      std::string element_topo = block.topologyType;
#ifdef IOSS_DEBUG_OUTPUT
      std::cout << "Added block " << block.name() << ":, IOSS topology = '" << element_topo
                << "' with " << block.ioss_count() << " elements\n";
#endif
      auto *eblock = new Ioss::ElementBlock(this, block.name(), element_topo, block.ioss_count());
      eblock->property_add(Ioss::Property("base", base));
      eblock->property_add(Ioss::Property("zone", block.zone()));
      eblock->property_add(Ioss::Property("id", block.zone()));
      eblock->property_add(Ioss::Property("section", block.section()));
      eblock->property_add(Ioss::Property("original_block_order", i++));
      get_region()->add(eblock);
    }

    // ========================================================================
    // Have sidesets, now create sideblocks for each sideset...
    int id = 0;
    for (auto &sset : decomp->m_sideSets) {
      // See if there is an Ioss::SideSet with a matching name...
      Ioss::SideSet *ioss_sset = get_region()->get_sideset(sset.name());
      if (ioss_sset != NULL) {
        auto        zone = decomp->m_zones[sset.zone()];
        std::string block_name(zone.m_name);
        block_name += "/";
        block_name += sset.name();
        std::string face_topo = sset.topologyType;
#ifdef IOSS_DEBUG_OUTPUT
        std::cout << "Processor " << myProcessor << ": Added sideblock " << block_name
                  << " of topo " << face_topo << " with " << sset.ioss_count() << " faces\n";
#endif
        const auto &block = decomp->m_elementBlocks[sset.parentBlockIndex];

        std::string      parent_topo = block.topologyType;
        Ioss::SideBlock *sblk =
            new Ioss::SideBlock(this, block_name, face_topo, parent_topo, sset.ioss_count());
        sblk->property_add(Ioss::Property("id", id));
        sblk->property_add(Ioss::Property("base", 1));
        sblk->property_add(Ioss::Property("zone", sset.zone()));
        sblk->property_add(Ioss::Property("section", sset.section()));
        Ioss::ElementBlock *eblock = get_region()->get_element_block(block.name());
        if (eblock != NULL) {
          sblk->set_parent_element_block(eblock);
        }
        ioss_sset->add(sblk);
      }
      id++; // Really just index into m_sideSets list.
    }

    Ioss::NodeBlock *nblock =
        new Ioss::NodeBlock(this, "nodeblock_1", decomp->ioss_node_count(), 3);
    nblock->property_add(Ioss::Property("base", base));
    get_region()->add(nblock);

    // Create a single node commset
    Ioss::CommSet *commset =
        new Ioss::CommSet(this, "commset_node", "node", decomp->get_commset_node_size());
    commset->property_add(Ioss::Property("id", 1));
    get_region()->add(commset);
  }

  // TODO: See if code can be used for parallel node resolution...
  size_t ParallelDatabaseIO::finalize_structured_blocks()
  {
    const auto &blocks = get_region()->get_structured_blocks();

    // If there are any Structured blocks, need to iterate them and their 1-to-1 connections
    // and update the donor_zone id for zones that had not yet been processed at the time of
    // definition...
    for (auto &block : blocks) {
      for (auto &conn : block->m_zoneConnectivity) {
        if (conn.m_donorZone < 0) {
          auto donor_iter = m_zoneNameMap.find(conn.m_donorName);
          assert(donor_iter != m_zoneNameMap.end());
          conn.m_donorZone = (*donor_iter).second;
        }
      }
    }

    size_t num_nodes = Utils::resolve_nodes(*get_region(), myProcessor, true);

    Utils::resolve_shared_nodes(*get_region(), myProcessor);

    return num_nodes;
  }

  void ParallelDatabaseIO::handle_structured_blocks()
  {
    int base = 1;

    Utils::add_sidesets(cgnsFilePtr, this);

    char     basename[33];
    cgsize_t cell_dimension = 0;
    cgsize_t phys_dimension = 0;
    CGCHECK(cg_base_read(cgnsFilePtr, base, basename, &cell_dimension, &phys_dimension));

    // Iterate all structured blocks and set the intervals to zero
    // if the m_proc field does not match current processor...
    const auto &zones = decomp->m_structuredZones;

    size_t node_offset        = 0;
    size_t cell_offset        = 0;
    size_t global_node_offset = 0;
    size_t global_cell_offset = 0;

    for (auto &zone : zones) {
      if (zone->m_adam == zone) {
        // This is a "root" zone from the undecomposed mesh...
        // Now see if there are any non-empty blocks with
        // this m_adam on this processor.  If exists, then create
        // a StructuredBlock; otherwise, create an empty block.
        auto block_name = zone->m_name;

        Ioss::StructuredBlock *block = nullptr;
        Ioss::IJK_t            zeros{{0, 0, 0}};
        for (auto &pzone : zones) {
          if (pzone->m_proc == myProcessor && pzone->m_adam == zone) {
            // Create a non-empty structured block on this processor...
            block = new Ioss::StructuredBlock(this, block_name, phys_dimension, pzone->m_ordinal,
                                              pzone->m_offset, pzone->m_adam->m_ordinal);

            for (auto &zgc : pzone->m_zoneConnectivity) {
              block->m_zoneConnectivity.push_back(zgc);
            }
            break;
          }
        }
        if (block == nullptr) {
          // There is no block on this processor corresponding to the m_adam
          // block.  Create an empty block...
          block = new Ioss::StructuredBlock(this, block_name, phys_dimension, zeros, zeros,
                                            zone->m_adam->m_ordinal);
          for (auto zgc : zone->m_zoneConnectivity) {
            zgc.m_isActive = false;
            block->m_zoneConnectivity.push_back(zgc);
          }
        }
        assert(block != nullptr);
        get_region()->add(block);

        block->property_add(Ioss::Property("base", base));
        block->property_add(Ioss::Property("zone", zone->m_adam->m_zone));
        block->property_add(Ioss::Property("id", zone->m_adam->m_zone));

        block->set_node_offset(node_offset);
        block->set_cell_offset(cell_offset);
        node_offset += block->get_property("node_count").get_int();
        cell_offset += block->get_property("cell_count").get_int();

        block->set_node_global_offset(global_node_offset);
        block->set_cell_global_offset(global_cell_offset);
        global_node_offset += block->get_property("global_node_count").get_int();
        global_cell_offset += block->get_property("global_cell_count").get_int();
      }
    }

    // ========================================================================
    // Iterate each StructuredBlock, get its zone. For that zone, get the number of
    // boundary conditions and then iterate those and create sideblocks in the
    // corresponding sideset.
    const auto &sbs = get_region()->get_structured_blocks();
    for (const auto &block : sbs) {
      // Handle boundary conditions...
      Utils::add_structured_boundary_conditions(cgnsFilePtr, block);
    }

    size_t node_count = finalize_structured_blocks();
    auto * nblock     = new Ioss::NodeBlock(this, "nodeblock_1", node_count, phys_dimension);
    nblock->property_add(Ioss::Property("base", base));
    get_region()->add(nblock);
  }

  void ParallelDatabaseIO::resolve_zone_shared_nodes(const CGNSIntVector &nodes,
                                                     CGNSIntVector &      connectivity_map,
                                                     size_t &             owned_node_count,
                                                     size_t &             owned_node_offset) const
  {
    // Determine number of processors that have nodes for this zone.
    // Avoid mpi_comm_split call if possible.
    int have_nodes             = nodes.empty() ? 0 : 1;
    int shared_zone_proc_count = 0;
    MPI_Allreduce(&have_nodes, &shared_zone_proc_count, 1, Ioss::mpi_type(int(0)), MPI_SUM,
                  util().communicator());

    if (shared_zone_proc_count <= 1) {
      // There are no shared nodes in this zone.
      owned_node_count = nodes.size();
      std::iota(connectivity_map.begin(), connectivity_map.end(), 1);
      return;
    }

    have_nodes = have_nodes == 0 ? MPI_UNDEFINED : 1;
    MPI_Comm pcomm;
    MPI_Comm_split(util().communicator(), have_nodes, myProcessor, &pcomm);

    if (have_nodes == 1) {
      // From here on down, only processors that have nodes are involved...
      // This zone has nodes/cells on two or more processors.  Need to determine
      // which nodes are shared.

      Ioss::ParallelUtils pm(pcomm);
      size_t              proc_count = pm.parallel_size();
      assert((int)proc_count == shared_zone_proc_count);

      // Distribute each node to an "owning" processor based on its id
      // and assuming a linear distribution (e.g., if on 3 processors, each
      // proc will "own" 1/3 of the id range.
      // nodes is sorted.
      int64_t min = nodes[0];
      int64_t max = nodes.back();
      min         = pm.global_minmax(min, Ioss::ParallelUtils::DO_MIN);
      max         = pm.global_minmax(max, Ioss::ParallelUtils::DO_MAX);

      std::vector<int> p_count(proc_count);
      size_t           per_proc = (max - min + proc_count) / proc_count;
      size_t           proc     = 0;
      int64_t          top      = min + per_proc;

      // NOTE: nodes is sorted...
      for (size_t i = 0; i < nodes.size(); i++) {
        while (nodes[i] >= top) {
          top += per_proc;
          proc++;
        }
        assert(proc < proc_count);
        p_count[proc]++;
      }

      // Tell each processor how many nodes it will be getting...
      // Each processor will be responsible for a subsetted range of the overall range.
      // This processor, should range from min + my_proc*per_proc to min + (my_proc+1)*per_proc.
      std::vector<int> r_count(proc_count);
      MPI_Alltoall(TOPTR(p_count), 1, Ioss::mpi_type(int(0)), TOPTR(r_count), 1,
                   Ioss::mpi_type(int(0)), pcomm);

      std::vector<int> recv_disp(proc_count);
      std::vector<int> send_disp(proc_count);
      size_t           sums = 0;
      size_t           sumr = 0;
      for (size_t p = 0; p < proc_count; p++) {
        recv_disp[p] = sumr;
        sumr += r_count[p];

        send_disp[p] = sums;
        sums += p_count[p];
      }
      CGNSIntVector r_nodes(sumr);
      Ioss::MY_Alltoallv(nodes, p_count, send_disp, r_nodes, r_count, recv_disp, pcomm);

      // Iterate r_nodes list to find duplicate nodes...
      auto             delta = min + pm.parallel_rank() * per_proc;
      std::vector<int> dup_nodes(per_proc);
      for (size_t i = 0; i < r_nodes.size(); i++) {
        auto n = r_nodes[i] - delta;
        assert(n < per_proc);
        dup_nodes[n]++;
        if (dup_nodes[n] > 1) {
          r_nodes[i] = 0;
        }
      }

      // Send filtered list back to original processors -- store in 'u_nodes'
      // This is set of unique block nodes owned by this processor.
      // If an entry in 'u_nodes' is 0, then that is a non-owned shared node.
      CGNSIntVector u_nodes(nodes.size());
      Ioss::MY_Alltoallv(r_nodes, r_count, recv_disp, u_nodes, p_count, send_disp, pcomm);

      // Count non-zero entries in u_nodes...
      int64_t local_node_count =
          std::count_if(u_nodes.begin(), u_nodes.end(), [](int64_t i) { return i > 0; });
      owned_node_count = local_node_count; // Calling code wants to know this

      // Determine offset into the zone node block for each processors "chunk"
      int64_t local_node_offset = 0;
      MPI_Exscan(&local_node_count, &local_node_offset, 1, Ioss::mpi_type(local_node_count),
                 MPI_SUM, pcomm);
      owned_node_offset = local_node_offset; // Calling code wants to know this

      // This generates the position of each owned node in this zone consistent
      // over all processors that this zone is active on.
      for (size_t i = 0; i < u_nodes.size(); i++) {
        if (u_nodes[i] > 0) {
          u_nodes[i] = ++local_node_offset; // 1-based local node id for all owned nodes.
        }
      }

      // u_nodes now contains the global -> block-local node map for all owned nodes
      // on the processor.
      // The zeroes in u_nodes are shared nodes on the processor boundary.
      // Resend nodes and u_nodes so can resolve the ids of the shared nodes.
      CGNSIntVector g_to_zone_local(sumr);
      Ioss::MY_Alltoallv(nodes, p_count, send_disp, r_nodes, r_count, recv_disp, pcomm);
      Ioss::MY_Alltoallv(u_nodes, p_count, send_disp, g_to_zone_local, r_count, recv_disp, pcomm);

      // Iterate g_to_zone_local to find a zero entry.
      for (size_t i = 0; i < g_to_zone_local.size(); i++) {
        if (g_to_zone_local[i] == 0) {
          // The global id is r_nodes[i] which must also appear earlier in the list...
          for (size_t j = 0; j < i; j++) {
            if (r_nodes[j] == r_nodes[i]) {
              g_to_zone_local[i] = g_to_zone_local[j];
              break;
            }
          }
        }
      }

      // Now, send updated g_to_zone_local back to original processors...
      Ioss::MY_Alltoallv(g_to_zone_local, r_count, recv_disp, u_nodes, p_count, send_disp, pcomm);

// At this point:
//   'nodes' contains the global node ids that are referenced in this zones connectivity.
//   'u_nodes' contains the zone-local 1-based position of that node in this zones node list.
//
//
#ifndef NDEBUG
      for (size_t i = 0; i < u_nodes.size(); i++) {
        assert(u_nodes[i] > 0);
      }
#endif
      std::swap(connectivity_map, u_nodes);
      MPI_Comm_free(&pcomm);
    }
  }

  void ParallelDatabaseIO::write_meta_data()
  {
    int num_zones = get_region()->get_property("element_block_count").get_int() +
                    get_region()->get_property("structured_block_count").get_int();
    m_bcOffset.resize(num_zones + 1);   // use 1-based zones...
    m_zoneOffset.resize(num_zones + 1); // use 1-based zones...

    Utils::common_write_meta_data(cgnsFilePtr, *get_region(), m_zoneOffset);
  }

  void ParallelDatabaseIO::get_step_times__()
  {
    Utils::get_step_times(cgnsFilePtr, m_timesteps, get_region(), timeScaleFactor, myProcessor);
  }

  void ParallelDatabaseIO::write_adjacency_data()
  {
#if 0
    // Determine adjacency information between unstructured blocks.
    // Could save this information from the input mesh, but then
    // could not read an exodus mesh and write a cgns mesh.
    // However, in long run may still want to read/save input adjacency
    // data if doing cgns -> cgns...  For now, try generating information.

    // If block I is adjacent to block J, then they will share at
    // least 1 "side" (face 3D or edge 2D).
    // Currently, assuming they are adjacent if they share at least one node...

    size_t node_count = get_region()->get_property("node_count").get_int();

    const auto &blocks = get_region()->get_element_blocks();
    for (auto I = blocks.begin(); I != blocks.end(); I++) {
      cgsize_t base = (*I)->get_property("base").get_int();
      cgsize_t zone = (*I)->get_property("zone").get_int();

      const auto &I_map = m_globalToBlockLocalNodeMap[zone];

      // Flag all nodes used by this block...
      std::vector<size_t> I_nodes(node_count);
      for (size_t i = 0; i < I_map->map().size() - 1; i++) {
        auto global     = I_map->map()[i + 1] - 1;
        I_nodes[global] = i + 1;
      }
      for (auto J = I + 1; J != blocks.end(); J++) {
        cgsize_t              dzone = (*J)->get_property("zone").get_int();
        const auto &          J_map = m_globalToBlockLocalNodeMap[dzone];
        CGNSIntVector point_list;
        CGNSIntVector point_list_donor;
        for (size_t i = 0; i < J_map->map().size() - 1; i++) {
          auto global = J_map->map()[i + 1] - 1;
          if (I_nodes[global] > 0) {
            // Have a match between nodes used by two different blocks,
            // They are adjacent...
            point_list.push_back(I_nodes[global]);
            point_list_donor.push_back(i + 1);
          }
        }

        // If point_list non_empty, then output this adjacency node...
        if (!point_list.empty()) {
          int         gc_idx = 0;
          std::string name   = (*I)->name();
          name += "_to_";
          name += (*J)->name();
          const auto &d1_name = (*J)->name();
	  CGCHECK(cg_conn_write(cgnsFilePtr, base, zone, name.c_str(), CG_Vertex, CG_Abutting1to1,
                            CG_PointList, point_list.size(), TOPTR(point_list), d1_name.c_str(),
                            CG_Unstructured, CG_PointListDonor, CG_DataTypeNull,
				point_list_donor.size(), TOPTR(point_list_donor), &gc_idx));

          name = (*J)->name();
          name += "_to_";
          name += (*I)->name();
          const auto &d2_name = (*I)->name();

          CGCHECK(cg_conn_write(cgnsFilePtr, base, dzone, name.c_str(), CG_Vertex, CG_Abutting1to1,
                               CG_PointList, point_list_donor.size(), TOPTR(point_list_donor),
                               d2_name.c_str(), CG_Unstructured, CG_PointListDonor, CG_DataTypeNull,
				point_list.size(), TOPTR(point_list), &gc_idx));
        }
      }
    }
#endif
  }

  bool ParallelDatabaseIO::begin__(Ioss::State /* state */) { return true; }

  bool ParallelDatabaseIO::end__(Ioss::State state)
  {
    // Transitioning out of state 'state'
    switch (state) {
    case Ioss::STATE_DEFINE_MODEL:
      if (!is_input() && open_create_behavior() != Ioss::DB_APPEND) {
        write_meta_data();
      }
      break;
    case Ioss::STATE_MODEL:
      if (!is_input() && open_create_behavior() != Ioss::DB_APPEND) {
        write_adjacency_data();
      }
      break;
    case Ioss::STATE_DEFINE_TRANSIENT:
      if (!is_input() && open_create_behavior() != Ioss::DB_APPEND) {
        write_results_meta_data();
      }
      break;
    default: // ignore everything else...
      break;
    }
    return true;
  }

  bool ParallelDatabaseIO::begin_state__(Ioss::Region *region, int state, double time)
  {
    if (is_input()) {
      return true;
    }
    Utils::write_flow_solution_metadata(cgnsFilePtr, get_region(), state,
                                        &m_currentVertexSolutionIndex,
                                        &m_currentCellCenterSolutionIndex);
    return true;
  }

  bool ParallelDatabaseIO::end_state__(Ioss::Region * /* region */, int state, double time)
  {
    if (!is_input()) {
      m_timesteps.push_back(time);
      assert(m_timesteps.size() == (size_t)state);
    }
    return true;
  }

  const Ioss::Map &ParallelDatabaseIO::get_map(entity_type type) const
  {
    if (m_zoneType == CG_Unstructured) {
      switch (type) {
      case entity_type::NODE: {
        size_t offset = decomp->decomp_node_offset();
        size_t count  = decomp->decomp_node_count();
        return get_map(nodeMap, nodeCount, offset, count, entity_type::NODE);
      }
      case entity_type::ELEM: {
        size_t offset = decomp->decomp_elem_offset();
        size_t count  = decomp->decomp_elem_count();
        return get_map(elemMap, elementCount, offset, count, entity_type::ELEM);
      }
      }
    }
    else {
      assert(1 == 0);
    }
    std::ostringstream errmsg;
    errmsg << "INTERNAL ERROR: Invalid map type. "
           << "Something is wrong in the Iocgns::ParallelDatabaseIO::get_map() function. "
           << "Please report.\n";
    IOSS_ERROR(errmsg);
  }

  const Ioss::Map &ParallelDatabaseIO::get_map(Ioss::Map &entity_map, int64_t entityCount,
                                               int64_t file_offset, int64_t file_count,
                                               entity_type type) const

  {
    // Allocate space for node number map and read it in...
    // Can be called multiple times, allocate 1 time only
    if (entity_map.map().empty()) {
      entity_map.map().resize(entityCount + 1);

      if (is_input()) {
        Ioss::MapContainer file_data(file_count);

        // For cgns, my file_data is just nodes from file_offset to file_offset+file_count
        std::iota(file_data.begin(), file_data.end(), file_offset + 1);

        if (type == entity_type::NODE)
          decomp->communicate_node_data(TOPTR(file_data), &entity_map.map()[1], 1);
        else if (type == entity_type::ELEM)
          decomp->communicate_element_data(TOPTR(file_data), &entity_map.map()[1], 1);

        // Check for sequential node map.
        // If not, build the reverse G2L node map...
        entity_map.map()[0] = -1;
        for (int64_t i = 1; i < entityCount + 1; i++) {
          if (i != entity_map.map()[i]) {
            entity_map.map()[0] = 1;
            break;
          }
        }

        entity_map.build_reverse_map();
      }
      else {
        // Output database; entity_map.map not set yet... Build a default map.
        for (int64_t i = 1; i < entityCount + 1; i++) {
          entity_map.map()[i] = i;
        }
        // Sequential map
        entity_map.map()[0] = -1;
      }
    }
    return entity_map;
  }

  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::Region * /* reg */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }

  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::NodeBlock *nb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    size_t num_to_get = field.verify(data_size);

    Ioss::Field::RoleType role = field.get_role();
    if (role == Ioss::Field::MESH) {
      if (field.get_name() == "mesh_model_coordinates_x" ||
          field.get_name() == "mesh_model_coordinates_y" ||
          field.get_name() == "mesh_model_coordinates_z" ||
          field.get_name() == "mesh_model_coordinates") {
        decomp->get_node_coordinates(cgnsFilePtr, (double *)data, field);
      }

      else if (field.get_name() == "ids") {
        // Map the local ids in this node block
        // (1...node_count) to global node ids.
        get_map(entity_type::NODE).map_implicit_data(data, field, num_to_get, 0);
      }
      // The 1..global_node_count id.  In a parallel-decomposed run,
      // it maps the node back to its implicit position in the serial
      // undecomposed mesh file.  This is ONLY provided for backward-
      // compatibility and should not be used unless absolutely required.
      else if (field.get_name() == "implicit_ids") {
        size_t offset = decomp->decomp_node_offset();
        size_t count  = decomp->decomp_node_count();
        if (int_byte_size_api() == 4) {
          std::vector<int> file_ids(count);
          std::iota(file_ids.begin(), file_ids.end(), offset + 1);
          decomp->communicate_node_data(TOPTR(file_ids), (int *)data, 1);
        }
        else {
          std::vector<int64_t> file_ids(count);
          std::iota(file_ids.begin(), file_ids.end(), offset + 1);
          decomp->communicate_node_data(TOPTR(file_ids), (int64_t *)data, 1);
        }
      }

      else if (field.get_name() == "connectivity") {
        // Do nothing, just handles an idiosyncracy of the GroupingEntity
      }
      else if (field.get_name() == "connectivity_raw") {
        // Do nothing, just handles an idiosyncracy of the GroupingEntity
      }
      else if (field.get_name() == "owning_processor") {
        // If parallel, then set the "locally_owned" property on the nodeblocks.
        Ioss::CommSet *css = get_region()->get_commset("commset_node");
        if (int_byte_size_api() == 8) {
          int64_t *idata = static_cast<int64_t *>(data);
          std::fill(idata, idata + nodeCount, myProcessor);

          std::vector<int64_t> ent_proc;
          css->get_field_data("entity_processor_raw", ent_proc);
          for (size_t i = 0; i < ent_proc.size(); i += 2) {
            int64_t node = ent_proc[i + 0];
            int64_t proc = ent_proc[i + 1];
            if (proc < idata[node - 1]) {
              idata[node - 1] = proc;
            }
          }
        }
        else {
          int *idata = static_cast<int *>(data);
          std::fill(idata, idata + nodeCount, myProcessor);

          std::vector<int> ent_proc;
          css->get_field_data("entity_processor_raw", ent_proc);
          for (size_t i = 0; i < ent_proc.size(); i += 2) {
            int node = ent_proc[i + 0];
            int proc = ent_proc[i + 1];
            if (proc < idata[node - 1]) {
              idata[node - 1] = proc;
            }
          }
        }
      }
      else {
        num_to_get = Ioss::Utils::field_warning(nb, field, "input");
      }
    }
    else if (role == Ioss::Field::TRANSIENT) {
      // Locate the FlowSolution node corresponding to the correct state/step/time
      // TODO: do this at read_meta_data() and store...
      int  step       = get_region()->get_current_state();
      auto var_type   = field.transformed_storage();
      int  comp_count = var_type->component_count();

      if (comp_count == 1) {
        decomp->get_node_field(cgnsFilePtr, step, Utils::index(field), (double *)data);
      }
      else {
        std::vector<double> ioss_tmp(num_to_get);
        for (int i = 0; i < comp_count; i++) {
          decomp->get_node_field(cgnsFilePtr, step, Utils::index(field) + i, ioss_tmp.data());

          size_t  index = i;
          double *rdata = static_cast<double *>(data);
          for (size_t j = 0; j < num_to_get; j++) {
            rdata[index] = ioss_tmp[j];
            index += comp_count;
          }
        }
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(nb, field, "input");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::EdgeBlock * /* nb */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::FaceBlock * /* nb */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }

  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::StructuredBlock *sb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    Ioss::Field::RoleType role = field.get_role();
    cgsize_t              base = sb->get_property("base").get_int();
    cgsize_t              zone = sb->get_property("zone").get_int();

    cgsize_t num_to_get = field.verify(data_size);

    cgsize_t rmin[3] = {0, 0, 0};
    cgsize_t rmax[3] = {0, 0, 0};

    bool cell_field = Utils::is_cell_field(field);
    if (cell_field) {
      assert(num_to_get == sb->get_property("cell_count").get_int());
      if (num_to_get > 0) {
        rmin[0] = sb->get_property("offset_i").get_int() + 1;
        rmin[1] = sb->get_property("offset_j").get_int() + 1;
        rmin[2] = sb->get_property("offset_k").get_int() + 1;

        rmax[0] = rmin[0] + sb->get_property("ni").get_int() - 1;
        rmax[1] = rmin[1] + sb->get_property("nj").get_int() - 1;
        rmax[2] = rmin[2] + sb->get_property("nk").get_int() - 1;
      }
    }
    else {
      // cell nodal field.
      assert(num_to_get == sb->get_property("node_count").get_int());
      if (num_to_get > 0) {
        rmin[0] = sb->get_property("offset_i").get_int() + 1;
        rmin[1] = sb->get_property("offset_j").get_int() + 1;
        rmin[2] = sb->get_property("offset_k").get_int() + 1;

        rmax[0] = rmin[0] + sb->get_property("ni").get_int();
        rmax[1] = rmin[1] + sb->get_property("nj").get_int();
        rmax[2] = rmin[2] + sb->get_property("nk").get_int();
      }
    }

    assert(num_to_get == 0 ||
           num_to_get ==
               (rmax[0] - rmin[0] + 1) * (rmax[1] - rmin[1] + 1) * (rmax[2] - rmin[2] + 1));
    double *rdata = num_to_get > 0 ? static_cast<double *>(data) : nullptr;

    if (role == Ioss::Field::MESH) {

      if (field.get_name() == "mesh_model_coordinates_x") {
        CGCHECK(cgp_coord_read_data(cgnsFilePtr, base, zone, 1, rmin, rmax, rdata));
      }

      else if (field.get_name() == "mesh_model_coordinates_y") {
        CGCHECK(cgp_coord_read_data(cgnsFilePtr, base, zone, 2, rmin, rmax, rdata));
      }

      else if (field.get_name() == "mesh_model_coordinates_z") {
        CGCHECK(cgp_coord_read_data(cgnsFilePtr, base, zone, 3, rmin, rmax, rdata));
      }

      else if (field.get_name() == "mesh_model_coordinates") {
        char     basename[33];
        cgsize_t cell_dimension = 0;
        cgsize_t phys_dimension = 0;
        CGCHECK(cg_base_read(cgnsFilePtr, base, basename, &cell_dimension, &phys_dimension));

        // Data required by upper classes store x0, y0, z0, ... xn,
        // yn, zn. Data stored in cgns file is x0, ..., xn, y0,
        // ..., yn, z0, ..., zn so we have to allocate some scratch
        // memory to read in the data and then map into supplied
        // 'data'
        std::vector<double> coord(num_to_get);
        CGCHECK(cgp_coord_read_data(cgnsFilePtr, base, zone, 1, rmin, rmax, TOPTR(coord)));

        // Map to global coordinate position...
        for (cgsize_t i = 0; i < num_to_get; i++) {
          rdata[phys_dimension * i + 0] = coord[i];
        }

        if (phys_dimension >= 2) {
          CGCHECK(cgp_coord_read_data(cgnsFilePtr, base, zone, 2, rmin, rmax, TOPTR(coord)));

          // Map to global coordinate position...
          for (cgsize_t i = 0; i < num_to_get; i++) {
            rdata[phys_dimension * i + 1] = coord[i];
          }
        }

        if (phys_dimension == 3) {
          CGCHECK(cgp_coord_read_data(cgnsFilePtr, base, zone, 3, rmin, rmax, TOPTR(coord)));

          // Map to global coordinate position...
          for (cgsize_t i = 0; i < num_to_get; i++) {
            rdata[phys_dimension * i + 2] = coord[i];
          }
        }
      }
      else if (field.get_name() == "cell_node_ids") {
        if (field.get_type() == Ioss::Field::INT64) {
          int64_t *idata = static_cast<int64_t *>(data);
          sb->get_cell_node_ids(idata, true);
        }
        else {
          assert(field.get_type() == Ioss::Field::INT32);
          int *idata = static_cast<int *>(data);
          sb->get_cell_node_ids(idata, true);
        }
      }
      else if (field.get_name() == "cell_ids") {
        if (field.get_type() == Ioss::Field::INT64) {
          int64_t *idata = static_cast<int64_t *>(data);
          sb->get_cell_ids(idata, true);
        }
        else {
          assert(field.get_type() == Ioss::Field::INT32);
          int *idata = static_cast<int *>(data);
          sb->get_cell_ids(idata, true);
        }
      }
      else {
        num_to_get = Ioss::Utils::field_warning(sb, field, "input");
      }
    }
    else if (role == Ioss::Field::TRANSIENT) {
      auto var_type   = field.transformed_storage();
      int  comp_count = var_type->component_count();

      int sol_index = 0;
      int step      = get_region()->get_current_state();
      if (cell_field) {
        sol_index = Utils::find_solution_index(cgnsFilePtr, base, zone, step, CG_CellCenter);
      }
      else {
        sol_index = Utils::find_solution_index(cgnsFilePtr, base, zone, step, CG_Vertex);
      }
      int field_offset = Utils::index(field);

      if (comp_count == 1) {
        CGCHECK(cgp_field_read_data(cgnsFilePtr, base, zone, sol_index, field_offset, rmin, rmax,
                                    rdata));
      }
      else {
        std::vector<double> cgns_data(num_to_get);
        for (int i = 0; i < comp_count; i++) {
          CGCHECK(cgp_field_read_data(cgnsFilePtr, base, zone, sol_index, field_offset + i, rmin,
                                      rmax, cgns_data.data()));
          for (cgsize_t j = 0; j < num_to_get; j++) {
            rdata[comp_count * j + i] = cgns_data[j];
          }
        }
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(sb, field, "input");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::ElementBlock *eb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    cgsize_t base       = eb->get_property("base").get_int();
    cgsize_t zone       = eb->get_property("zone").get_int();
    size_t   num_to_get = field.verify(data_size);
    auto     role       = field.get_role();

    if (role == Ioss::Field::MESH) {
      // Handle the MESH fields required for a CGNS file model.
      // (The 'genesis' portion)

      if (field.get_name() == "connectivity_raw" || field.get_name() == "connectivity") {

        // The connectivity is stored in a 1D array.
        // The element_node index varies fastet
        int order = eb->get_property("original_block_order").get_int();
        decomp->get_block_connectivity(cgnsFilePtr, data, order);
      }
      else if (field.get_name() == "ids" || field.get_name() == "implicit_ids") {
        // Map the local ids in this node block
        // (1...node_count) to global node ids.
        get_map(entity_type::ELEM).map_implicit_data(data, field, num_to_get, eb->get_offset());
      }
      else {
        num_to_get = Ioss::Utils::field_warning(eb, field, "input");
      }
    }
    else if (role == Ioss::Field::TRANSIENT) {
      // Locate the FlowSolution node corresponding to the correct state/step/time
      // TODO: do this at read_meta_data() and store...
      int step           = get_region()->get_current_state();
      int solution_index = Utils::find_solution_index(cgnsFilePtr, base, zone, step, CG_CellCenter);

      int order = eb->get_property("original_block_order").get_int();

      const Ioss::VariableType *var_type = field.raw_storage();

      // Read into a double variable
      // TODO: Support other field types...
      size_t              num_entity = eb->get_property("entity_count").get_int();
      std::vector<double> temp(num_entity);

      // get number of components, cycle through each component
      size_t comp_count = var_type->component_count();
      for (size_t i = 0; i < comp_count; i++) {
        int field_offset = Utils::index(field) + i;
        decomp->get_element_field(cgnsFilePtr, solution_index, order, field_offset, temp.data());

        // Transfer to 'data' array.
        size_t k = 0;
        assert(field.get_type() == Ioss::Field::REAL);
        double *rvar = static_cast<double *>(data);
        for (size_t j = i; j < num_entity * comp_count; j += comp_count) {
          rvar[j] = temp[k++];
        }
        assert(k == num_entity);
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(eb, field, "unknown");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::NodeSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::EdgeSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::FaceSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::ElementSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::SideBlock *sb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    int  id   = sb->get_property("id").get_int();
    auto sset = decomp->m_sideSets[id];

    ssize_t num_to_get = field.verify(data_size);
    if (num_to_get > 0) {
      int64_t entity_count = sb->get_property("entity_count").get_int();
      if (num_to_get != entity_count) {
        std::ostringstream errmsg;
        errmsg << "ERROR: Partial field input not yet implemented for side blocks";
        IOSS_ERROR(errmsg);
      }
    }

    Ioss::Field::RoleType role = field.get_role();
    if (role == Ioss::Field::MESH) {
      if (field.get_name() == "element_side_raw" || field.get_name() == "element_side") {

        decomp->get_sideset_element_side(cgnsFilePtr, sset, data);

        if (field.get_type() == Ioss::Field::INT32) {
          int *idata = (int *)data;
          Utils::map_cgns_face_to_ioss(sb->parent_element_topology(), num_to_get, idata);
        }
        else {
          int64_t *idata = (int64_t *)data;
          Utils::map_cgns_face_to_ioss(sb->parent_element_topology(), num_to_get, idata);
        }
        return num_to_get;
      }
      else {
        num_to_get = Ioss::Utils::field_warning(sb, field, "input");
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(sb, field, "input");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::SideSet * /* fs */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::get_field_internal(const Ioss::CommSet *cs, const Ioss::Field &field,
                                                 void *data, size_t data_size) const
  {
    size_t num_to_get = field.verify(data_size);

    // Return the <entity (node or side), processor> pair
    if (field.get_name() == "entity_processor" || field.get_name() == "entity_processor_raw") {

      // Check type -- node or side
      std::string type = cs->get_property("entity_type").get_string();

      if (type == "node") {

        bool do_map = field.get_name() == "entity_processor";
        // Convert local node id to global node id and store in 'data'
        const Ioss::MapContainer &map = get_map(entity_type::NODE).map();
        if (int_byte_size_api() == 4) {
          decomp->get_node_entity_proc_data(static_cast<int *>(data), map, do_map);
        }
        else {
          decomp->get_node_entity_proc_data(static_cast<int64_t *>(data), map, do_map);
        }
      }
      else {
        std::ostringstream errmsg;
        errmsg << "ERROR: Invalid commset type " << type;
        IOSS_ERROR(errmsg);
      }
    }
    else if (field.get_name() == "ids") {
      // Do nothing, just handles an idiosyncracy of the GroupingEntity
    }
    else {
      num_to_get = Ioss::Utils::field_warning(cs, field, "input");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::Region *region,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    return -1;
  }

  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::NodeBlock *nb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    Ioss::Field::RoleType role       = field.get_role();
    cgsize_t              base       = 1;
    size_t                num_to_get = field.verify(data_size);

    // Instead of outputting a global nodeblock's worth of data,
    // the data is output a "zone" at a time.
    // The m_globalToBlockLocalNodeMap[zone] map is used (Ioss::Map pointer)
    // This map is built during the output of block connectivity,
    // so for cgns unstructured mesh, we need to output ElementBlock connectivity
    // prior to outputting nodal coordinates.
    for (const auto &z : m_globalToBlockLocalNodeMap) {
      if (z.second == nullptr) {
        std::ostringstream errmsg;
        errmsg << "ERROR: CGNS: The globalToBlockLocalNodeMap is not defined, so nodal fields "
                  "cannot be output.";
        IOSS_ERROR(errmsg);
      }
    }

    if (role == Ioss::Field::MESH) {
      if (field.get_name() == "ids") {
        // The ids coming in are the global ids; their position is the
        // local id -1 (That is, data[0] contains the global id of local
        // node 1)

        // Another 'const-cast' since we are modifying the database just
        // for efficiency; which the client does not see...
        handle_node_ids(data, num_to_get);
      }
      if (field.get_name() == "mesh_model_coordinates" ||
          field.get_name() == "mesh_model_coordinates_x" ||
          field.get_name() == "mesh_model_coordinates_y" ||
          field.get_name() == "mesh_model_coordinates_z") {
        double *rdata = static_cast<double *>(data);

        size_t               num_zones = m_globalToBlockLocalNodeMap.size();
        std::vector<int64_t> node_count(num_zones);
        std::vector<int64_t> node_offset(num_zones);

        for (const auto &block : m_globalToBlockLocalNodeMap) {
          auto        zone      = block.first;
          const auto &block_map = block.second;
          node_count[zone - 1]  = block_map->map().size() - 1;
        }
        MPI_Exscan(TOPTR(node_count), TOPTR(node_offset), num_zones, Ioss::mpi_type(node_count[0]),
                   MPI_SUM, util().communicator());

        if (field.get_name() == "mesh_model_coordinates") {
          int spatial_dim = nb->get_property("component_degree").get_int();

          // Output all coordinates, a zone's worth of data at a time...

          for (const auto &block : m_globalToBlockLocalNodeMap) {
            auto zone = block.first;
            // NOTE: 'block_map' has one more entry than node_count.  First entry is for something
            // else.
            //       'block_map' is 1-based.
            const auto &        block_map = block.second;
            std::vector<double> x(block_map->map().size() - 1);
            std::vector<double> y(block_map->map().size() - 1);
            std::vector<double> z(block_map->map().size() - 1);

            for (size_t i = 0; i < block_map->map().size() - 1; i++) {
              auto global = block_map->map()[i + 1];
              auto local  = nodeMap.global_to_local(global) - 1;
              assert(local >= 0 && local < (int64_t)num_to_get);

              x[i] = rdata[local * spatial_dim + 0];
              if (spatial_dim > 1) {
                y[i] = rdata[local * spatial_dim + 1];
              }
              if (spatial_dim > 2) {
                z[i] = rdata[local * spatial_dim + 2];
              }
            }

            // Create the zone
            // Output this zones coordinates...
            int crd_idx = 0;
            CGCHECK(
                cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, "CoordinateX", &crd_idx));
            cgsize_t start  = node_offset[zone - 1] + 1;
            cgsize_t finish = start + node_count[zone - 1] - 1;

            auto xx = node_count[zone - 1] > 0 ? TOPTR(x) : nullptr;
            CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, crd_idx, &start, &finish, xx));

            if (spatial_dim > 1) {
              CGCHECK(
                  cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, "CoordinateY", &crd_idx));
              auto yy = node_count[zone - 1] > 0 ? TOPTR(y) : nullptr;
              CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, crd_idx, &start, &finish, yy));
            }

            if (spatial_dim > 2) {
              CGCHECK(
                  cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, "CoordinateZ", &crd_idx));
              auto zz = node_count[zone - 1] > 0 ? TOPTR(z) : nullptr;
              CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, crd_idx, &start, &finish, zz));
            }
          }
        }
        else {
          // Outputting only a single coordinate value...
          for (const auto &block : m_globalToBlockLocalNodeMap) {
            auto zone = block.first;
            // NOTE: 'block_map' has one more entry than node_count.  First entry is for something
            // else.
            //       'block_map' is 1-based.
            const auto &        block_map = block.second;
            std::vector<double> xyz(block_map->map().size() - 1);

            for (size_t i = 0; i < block_map->map().size() - 1; i++) {
              auto global = block_map->map()[i + 1];
              auto local  = nodeMap.global_to_local(global) - 1;
              xyz[i]      = rdata[local];
            }

            std::string cgns_name = "Invalid";
            if (field.get_name() == "mesh_model_coordinates_x") {
              cgns_name = "CoordinateX";
            }
            else if (field.get_name() == "mesh_model_coordinates_y") {
              cgns_name = "CoordinateY";
            }
            else if (field.get_name() == "mesh_model_coordinates_z") {
              cgns_name = "CoordinateZ";
            }
            // Create the zone
            // Output this zones coordinates...
            int crd_idx = 0;
            CGCHECK(cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, cgns_name.c_str(),
                                    &crd_idx));
            cgsize_t start  = node_offset[zone - 1] + 1;
            cgsize_t finish = start + node_count[zone - 1] - 1;
            auto     xx     = node_count[zone - 1] > 0 ? TOPTR(xyz) : nullptr;
            CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, crd_idx, &start, &finish, xx));
          }
        }
      }
      else {
        num_to_get = Ioss::Utils::field_warning(nb, field, "input");
      }
    }
    else if (role == Ioss::Field::TRANSIENT) {
      // Instead of outputting a global nodeblock's worth of data,
      // the data is output a "zone" at a time.
      // The m_globalToBlockLocalNodeMap[zone] map is used (Ioss::Map pointer)
      // This map is built during the output of block connectivity,
      // so for cgns unstructured mesh, we need to output ElementBlock connectivity
      // prior to outputting nodal coordinates.
      size_t                num_zones = m_globalToBlockLocalNodeMap.size();
      std::vector<cgsize_t> node_count(num_zones);
      std::vector<cgsize_t> node_offset(num_zones);

      for (const auto &block : m_globalToBlockLocalNodeMap) {
        auto        zone      = block.first;
        const auto &block_map = block.second;
        node_count[zone - 1]  = block_map->map().size() - 1;
      }
      MPI_Exscan(TOPTR(node_count), TOPTR(node_offset), num_zones, Ioss::mpi_type(node_count[0]),
                 MPI_SUM, util().communicator());

      const Ioss::VariableType *var_type   = field.raw_storage();
      size_t                    comp_count = var_type->component_count();

      double *rdata = num_to_get > 0 ? static_cast<double *>(data) : nullptr;

      for (const auto &block : m_globalToBlockLocalNodeMap) {
        auto zone = block.first;
        // NOTE: 'block_map' has one more entry than node_count.
        // First entry is for something else.  'block_map' is
        // 1-based.
        const auto &        block_map = block.second;
        std::vector<double> blk_data(block_map->map().size() - 1);

        cgsize_t range_min[1] = {node_offset[zone - 1] + 1};
        cgsize_t range_max[1] = {range_min[0] + node_count[zone - 1] - 1};
        int      cgns_field   = 0;

        if (comp_count > 1) {
          char field_suffix_separator = get_field_separator();

          for (size_t i = 0; i < comp_count; i++) {
            for (size_t j = 0; j < block_map->map().size() - 1; j++) {
              auto global = block_map->map()[j + 1] - 1;
              blk_data[j] = rdata[comp_count * global + i];
            }
            std::string var_name =
                var_type->label_name(field.get_name(), i + 1, field_suffix_separator);
            CGCHECK(cgp_field_write(cgnsFilePtr, base, zone, m_currentVertexSolutionIndex,
                                    CG_RealDouble, var_name.c_str(), &cgns_field));

            CGCHECK(cgp_field_write_data(cgnsFilePtr, base, zone, m_currentVertexSolutionIndex,
                                         cgns_field, range_min, range_max, blk_data.data()));
            if (i == 0)
              Utils::set_field_index(field, cgns_field, CG_Vertex);
          }
        }
        else {
          CGCHECK(cgp_field_write(cgnsFilePtr, base, zone, m_currentVertexSolutionIndex,
                                  CG_RealDouble, field.get_name().c_str(), &cgns_field));

          CGCHECK(cgp_field_write_data(cgnsFilePtr, base, zone, m_currentVertexSolutionIndex,
                                       cgns_field, range_min, range_max, rdata));
          Utils::set_field_index(field, cgns_field, CG_CellCenter);
        }
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(nb, field, "input");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::ElementBlock *eb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    cgsize_t num_to_get = field.verify(data_size);

    Ioss::Field::RoleType role = field.get_role();

    if (role == Ioss::Field::MESH) {
      // Handle the MESH fields required for a CGNS file model.
      // (The 'genesis' portion)
      if (field.get_name() == "connectivity") {
        // This blocks zone has not been defined.
        // Get the "node block" for this element block...
        cgsize_t *idata         = num_to_get > 0 ? reinterpret_cast<cgsize_t *>(data) : nullptr;
        size_t    element_nodes = eb->topology()->number_nodes();
        assert((size_t)field.raw_storage()->component_count() == element_nodes);

        CGNSIntVector nodes;
        nodes.reserve(element_nodes * num_to_get);
        for (size_t i = 0; i < element_nodes * num_to_get; i++) {
          nodes.push_back(idata[i]);
        }
	Ioss::Utils::uniquify(nodes);

        // Resolve zone-shared nodes (nodes used in this zone, but are
        // shared on processor boundaries).
        // This routine determines the mapping of each global id node
        // in 'nodes' to the zone-local position.
        // This mapping is in 'connectivity_map' and is correct for all
        // nodes on this processor whether they are owned or shared.
        //
        // 'resolve_zone_shared_nodes' also returns the number of nodes owned on this
        // processor, and the 'offset' of this processors chunk of nodes into the overall
        // set of nodes for the zone.  Each processors chunk of nodes is contiguous
        //
        // The 'nodes' and 'connectivity_map' vectors are used later below to generate
        // the map of which global node data is written by this processor for this zone.
        CGNSIntVector connectivity_map(nodes.size());
        size_t        owned_node_count  = 0;
        size_t        owned_node_offset = 0;
        resolve_zone_shared_nodes(nodes, connectivity_map, owned_node_count, owned_node_offset);

        // Get total count on all processors...
        // Note that there will be shared nodes on processor boundaries that need to be
        // accounted for...
        cgsize_t size[3] = {0, 0, 0};
        size[0]          = owned_node_count;
        size[1]          = eb->get_property("entity_count").get_int();

        MPI_Allreduce(MPI_IN_PLACE, size, 3, Ioss::mpi_type(cgsize_t(0)), MPI_SUM,
                      util().communicator());

        // Now, we have the node count and cell count so we can create a zone...
        int base = 1;
        int zone = 0;

        CGCHECK(cg_zone_write(cgnsFilePtr, base, eb->name().c_str(), size, CG_Unstructured, &zone));
        eb->property_update("zone", zone);
        eb->property_update("id", zone);
        eb->property_update("section", 1);
        eb->property_update("base", base);

        if (size[1] > 0) {
          CG_ElementType_t type = Utils::map_topology_to_cgns(eb->topology()->name());
          int              sect = 0;
          CGCHECK(cgp_section_write(cgnsFilePtr, base, zone, "HexElements", type, 1, size[1], 0,
                                    &sect));

          cgsize_t start = 0;
          MPI_Exscan(&num_to_get, &start, 1, Ioss::mpi_type(cgsize_t(0)), MPI_SUM,
                     util().communicator());
          // Of the cells/elements in this zone, this proc handles
          // those starting at 'proc_offset+1' to 'proc_offset+num_entity'
          eb->property_update("proc_offset", start);

          // Map connectivity global ids to zone-local 1-based ids.
          for (size_t i = 0; i < num_to_get * element_nodes; i++) {
            auto id   = idata[i];
            auto iter = std::lower_bound(nodes.begin(), nodes.end(), id);
            assert(iter != nodes.end());
            auto cur_pos = iter - nodes.begin();
            idata[i]     = connectivity_map[cur_pos];
          }

          CGCHECK(cgp_elements_write_data(cgnsFilePtr, base, zone, sect, start + 1,
                                          start + num_to_get, idata));

          cgsize_t eb_size = num_to_get;
          MPI_Allreduce(MPI_IN_PLACE, &eb_size, 1, Ioss::mpi_type(cgsize_t(0)), MPI_SUM,
                        util().communicator());

          m_bcOffset[zone] += eb_size;
          eb->property_update("section", sect);
        }

        // The 'nodes' map needs to be updated to filter out any nodes
        // that are not owned by this processor.  Currently contains both
        // owned and shared so we could update the connectivity...
        // The 'connectivity_map' value indicates whether it is owned or shared --
        // if 'connectivity_map[i] > owned_node_offset, then it is owned; otherwise shared.
        if (!nodes.empty()) {
          for (size_t i = 0; i < nodes.size(); i++) {
            if (connectivity_map[i] <= (cgsize_t)owned_node_offset) {
              nodes[i] = std::numeric_limits<cgsize_t>::max();
            }
          }
          connectivity_map.clear();
          connectivity_map.shrink_to_fit();

	  Ioss::Utils::uniquify(nodes);
          if (nodes.back() == std::numeric_limits<cgsize_t>::max()) {
            nodes.pop_back();
          }
          nodes.shrink_to_fit();
        }
        assert(nodes.size() == owned_node_count);

        // Now we have a valid zone so can update some data structures...
        m_zoneOffset[zone]                = m_zoneOffset[zone - 1] + size[1];
        m_globalToBlockLocalNodeMap[zone] = new Ioss::Map("element", "unknown", myProcessor);
        m_globalToBlockLocalNodeMap[zone]->map().reserve(nodes.size() + 1);
        m_globalToBlockLocalNodeMap[zone]->map().push_back(1); // Non one-to-one map
        for (auto i : nodes) {
          m_globalToBlockLocalNodeMap[zone]->map().push_back(i);
        }
      }
      else {
        num_to_get = Ioss::Utils::field_warning(eb, field, "output");
      }
    }
    else if (role == Ioss::Field::TRANSIENT) {
      double *                  rdata    = num_to_get > 0 ? static_cast<double *>(data) : nullptr;
      const Ioss::VariableType *var_type = field.raw_storage();

      cgsize_t base = eb->get_property("base").get_int();
      cgsize_t zone = eb->get_property("zone").get_int();

      cgsize_t start        = eb->get_property("proc_offset").get_int();
      cgsize_t range_min[1] = {start + 1};
      cgsize_t range_max[1] = {start + num_to_get};

      // get number of components, cycle through each component
      size_t comp_count = var_type->component_count();
      if (comp_count == 1) {
        int cgns_field = 0;
        CGCHECK(cgp_field_write(cgnsFilePtr, base, zone, m_currentCellCenterSolutionIndex,
                                CG_RealDouble, field.get_name().c_str(), &cgns_field));
        CGCHECK(cgp_field_write_data(cgnsFilePtr, base, zone, m_currentCellCenterSolutionIndex,
                                     cgns_field, range_min, range_max, rdata));
        Utils::set_field_index(field, cgns_field, CG_CellCenter);
      }
      else {
        char                field_suffix_separator = get_field_separator();
        std::vector<double> cgns_data(num_to_get);
        for (size_t i = 0; i < comp_count; i++) {
          for (cgsize_t j = 0; j < num_to_get; j++) {
            cgns_data[j] = rdata[comp_count * j + i];
          }
          std::string var_name =
              var_type->label_name(field.get_name(), i + 1, field_suffix_separator);
          int cgns_field = 0;
          CGCHECK(cgp_field_write(cgnsFilePtr, base, zone, m_currentCellCenterSolutionIndex,
                                  CG_RealDouble, var_name.c_str(), &cgns_field));
          CGCHECK(cgp_field_write_data(cgnsFilePtr, base, zone, m_currentCellCenterSolutionIndex,
                                       cgns_field, range_min, range_max, cgns_data.data()));
          if (i == 0)
            Utils::set_field_index(field, cgns_field, CG_CellCenter);
        }
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(eb, field, "unknown");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::StructuredBlock *sb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    Ioss::Field::RoleType role = field.get_role();
    cgsize_t              base = sb->get_property("base").get_int();
    cgsize_t              zone = sb->get_property("zone").get_int();

    cgsize_t num_to_get = field.verify(data_size);

    cgsize_t rmin[3] = {0, 0, 0};
    cgsize_t rmax[3] = {0, 0, 0};

    bool cell_field = Utils::is_cell_field(field);

    if (cell_field) {
      assert(num_to_get == sb->get_property("cell_count").get_int());
      if (num_to_get > 0) {
        rmin[0] = sb->get_property("offset_i").get_int() + 1;
        rmin[1] = sb->get_property("offset_j").get_int() + 1;
        rmin[2] = sb->get_property("offset_k").get_int() + 1;

        rmax[0] = rmin[0] + sb->get_property("ni").get_int() - 1;
        rmax[1] = rmin[1] + sb->get_property("nj").get_int() - 1;
        rmax[2] = rmin[2] + sb->get_property("nk").get_int() - 1;
      }
    }
    else {
      // cell nodal field.
      assert(num_to_get == sb->get_property("node_count").get_int());
      if (num_to_get > 0) {
        rmin[0] = sb->get_property("offset_i").get_int() + 1;
        rmin[1] = sb->get_property("offset_j").get_int() + 1;
        rmin[2] = sb->get_property("offset_k").get_int() + 1;

        rmax[0] = rmin[0] + sb->get_property("ni").get_int();
        rmax[1] = rmin[1] + sb->get_property("nj").get_int();
        rmax[2] = rmin[2] + sb->get_property("nk").get_int();
      }
    }

    assert(num_to_get == 0 ||
           num_to_get ==
               (rmax[0] - rmin[0] + 1) * (rmax[1] - rmin[1] + 1) * (rmax[2] - rmin[2] + 1));
    double *rdata = num_to_get > 0 ? static_cast<double *>(data) : nullptr;

    if (role == Ioss::Field::MESH) {
      int crd_idx = 0;
      if (field.get_name() == "mesh_model_coordinates_x") {
        CGCHECK(cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, "CoordinateX", &crd_idx));
        CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, crd_idx, rmin, rmax, rdata));
      }

      else if (field.get_name() == "mesh_model_coordinates_y") {
        CGCHECK(cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, "CoordinateY", &crd_idx));
        CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, crd_idx, rmin, rmax, rdata));
      }

      else if (field.get_name() == "mesh_model_coordinates_z") {
        CGCHECK(cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, "CoordinateZ", &crd_idx));
        CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, crd_idx, rmin, rmax, rdata));
      }

      else if (field.get_name() == "mesh_model_coordinates") {
        int phys_dimension = get_region()->get_property("spatial_dimension").get_int();

        std::vector<double> coord(num_to_get);

        // ========================================================================
        // Repetitive code for each coordinate direction; use a lambda to consolidate...
        auto coord_lambda = [=, &coord](const char *ordinate, int ordinal) {
          // Data required by upper classes store x0, y0, z0, ... xn,
          // yn, zn. Data stored in cgns file is x0, ..., xn, y0,
          // ..., yn, z0, ..., zn so we have to allocate some scratch
          // memory to read in the data and then map into supplied
          // 'data'
          // Map to global coordinate position...
          for (cgsize_t i = 0; i < num_to_get; i++) {
            coord[i] = rdata[phys_dimension * i + ordinal];
          }

          int idx = 0;
          CGCHECK(cgp_coord_write(cgnsFilePtr, base, zone, CG_RealDouble, ordinate, &idx));
          CGCHECK(cgp_coord_write_data(cgnsFilePtr, base, zone, idx, rmin, rmax, TOPTR(coord)));
        };
        // ========================================================================

        coord_lambda("CoordinateX", 0);

        if (phys_dimension >= 2) {
          coord_lambda("CoordinateY", 1);
        }

        if (phys_dimension == 3) {
          coord_lambda("CoordinateZ", 2);
        }
      }
      else {
        num_to_get = Ioss::Utils::field_warning(sb, field, "output");
      }
    }
    else if (role == Ioss::Field::TRANSIENT) {
      int  cgns_field             = 0;
      auto var_type               = field.transformed_storage();
      int  comp_count             = var_type->component_count();
      char field_suffix_separator = get_field_separator();

      int               sol_index = 0;
      CG_GridLocation_t location;
      if (cell_field) {
        sol_index = m_currentCellCenterSolutionIndex;
        location  = CG_CellCenter;
      }
      else {
        sol_index = m_currentVertexSolutionIndex;
        location  = CG_Vertex;
      }
      if (comp_count == 1) {
        CGCHECK(cgp_field_write(cgnsFilePtr, base, zone, sol_index, CG_RealDouble,
                                field.get_name().c_str(), &cgns_field));
        Utils::set_field_index(field, cgns_field, location);

        CGCHECK(cgp_field_write_data(cgnsFilePtr, base, zone, sol_index, cgns_field, rmin, rmax,
                                     rdata));
      }
      else {
        std::vector<double> cgns_data(num_to_get);
        for (int i = 0; i < comp_count; i++) {
          for (cgsize_t j = 0; j < num_to_get; j++) {
            cgns_data[j] = rdata[comp_count * j + i];
          }
          std::string var_name =
              var_type->label_name(field.get_name(), i + 1, field_suffix_separator);

          CGCHECK(cgp_field_write(cgnsFilePtr, base, zone, sol_index, CG_RealDouble,
                                  var_name.c_str(), &cgns_field));
          if (i == 0) {
            Utils::set_field_index(field, cgns_field, location);
          }

          CGCHECK(cgp_field_write_data(cgnsFilePtr, base, zone, sol_index, cgns_field, rmin, rmax,
                                       rdata));
        }
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(sb, field, "output");
    }
    return num_to_get;
  }

  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::FaceBlock * /* nb */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::EdgeBlock * /* nb */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::NodeSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::EdgeSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::FaceSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::ElementSet * /* ns */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::SideBlock *sb,
                                                 const Ioss::Field &field, void *data,
                                                 size_t data_size) const
  {
    const Ioss::EntityBlock *parent_block = sb->parent_block();
    if (parent_block == nullptr) {
      std::ostringstream errmsg;
      errmsg << "ERROR: CGNS: SideBlock " << sb->name()
             << " does not have a parent-block specified.  This is required for CGNS output.";
      IOSS_ERROR(errmsg);
    }

    cgsize_t base       = parent_block->get_property("base").get_int();
    cgsize_t zone       = parent_block->get_property("zone").get_int();
    ssize_t  num_to_get = field.verify(data_size);

    Ioss::Field::RoleType role = field.get_role();

    if (role == Ioss::Field::MESH) {
      // Handle the MESH fields required for a CGNS file model.
      // (The 'genesis' portion)
      if (field.get_name() == "element_side") {
        // Get name from parent sideset...
        auto &name = sb->owner()->name();

        CG_ElementType_t type = Utils::map_topology_to_cgns(sb->topology()->name());
        cgsize_t         sect = 0;

        cgsize_t size = num_to_get;
        MPI_Allreduce(MPI_IN_PLACE, &size, 1, Ioss::mpi_type(cgsize_t(0)), MPI_SUM,
                      util().communicator());

        int cg_start = m_bcOffset[zone] + 1;
        int cg_end   = m_bcOffset[zone] + size;

        // NOTE: Currently not writing the "ElementConnectivity" data for the
        //       boundary condition.  It isn't used in the read and don't have
        //       the data so would have to generate it.  This may cause problems
        //       with codes that use the downstream data if they base the BC off
        //       of the nodes instead of the element/side info.
        CGCHECK(cgp_section_write(cgnsFilePtr, base, zone, name.c_str(), type, cg_start, cg_end, 0,
                                  &sect));

        sb->property_update("section", sect);

        size_t        offset = m_zoneOffset[zone - 1];
        CGNSIntVector parent(4 * num_to_get);

        if (field.get_type() == Ioss::Field::INT32) {
          int *  idata = reinterpret_cast<int *>(data);
          size_t j     = 0;
          for (ssize_t i = 0; i < num_to_get; i++) {
            parent[num_to_get * 0 + i] = idata[j++] - offset; // Element
            parent[num_to_get * 2 + i] = idata[j++];
          }
          // Adjust face numbers to IOSS convention instead of CGNS convention...
          Utils::map_ioss_face_to_cgns(sb->parent_element_topology(), num_to_get, parent);
        }
        else {
          int64_t *idata = reinterpret_cast<int64_t *>(data);
          size_t   j     = 0;
          for (ssize_t i = 0; i < num_to_get; i++) {
            parent[num_to_get * 0 + i] = idata[j++] - offset; // Element
            parent[num_to_get * 2 + i] = idata[j++];
          }
          // Adjust face numbers to IOSS convention instead of CGNS convention...
          Utils::map_ioss_face_to_cgns(sb->parent_element_topology(), num_to_get, parent);
        }

        int64_t local_face_count  = num_to_get;
        int64_t local_face_offset = 0;
        MPI_Exscan(&local_face_count, &local_face_offset, 1, Ioss::mpi_type(local_face_count),
                   MPI_SUM, util().communicator());
        cg_start = m_bcOffset[zone] + local_face_offset + 1;
        cg_end   = cg_start + local_face_count - 1;

        auto xx = num_to_get > 0 ? TOPTR(parent) : nullptr;
        CGCHECK(cgp_parent_data_write(cgnsFilePtr, base, zone, sect, cg_start, cg_end, xx));
        m_bcOffset[zone] += size;
      }
      else {
        num_to_get = Ioss::Utils::field_warning(sb, field, "output");
      }
    }
    else {
      num_to_get = Ioss::Utils::field_warning(sb, field, "output");
    }
    return num_to_get;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::SideSet * /* fs */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }
  int64_t ParallelDatabaseIO::put_field_internal(const Ioss::CommSet * /* cs */,
                                                 const Ioss::Field & /* field */, void * /* data */,
                                                 size_t /* data_size */) const
  {
    return -1;
  }

  void ParallelDatabaseIO::write_results_meta_data() {}

  unsigned ParallelDatabaseIO::entity_field_support() const { return Ioss::REGION; }

  int64_t ParallelDatabaseIO::handle_node_ids(void *ids, int64_t num_to_get) const
  {
    /*!
     * There are two modes we need to support in this routine:
     * 1. Initial definition of node map (local->global) and
     * nodeMap.reverse (global->local).
     * 2. Redefinition of node map via 'reordering' of the original
     * map when the nodes on this processor are the same, but their
     * order is changed (or count because of ghosting)
     *
     * So, there will be two maps the 'nodeMap.map' map is a 'direct lookup'
     * map which maps current local position to global id and the
     * 'nodeMap.reverse' is an associative lookup which maps the
     * global id to 'original local'.  There is also a
     * 'nodeMap.reorder' which is direct lookup and maps current local
     * position to original local.

     * The ids coming in are the global ids; their position is the
     * "local id-1" (That is, data[0] contains the global id of local
     * node 1 in this node block).
     *
     * int local_position = nodeMap.reverse[NodeMap[i+1]]
     * (the nodeMap.map and nodeMap.reverse are 1-based)
     *
     * To determine which map to update on a call to this function, we
     * use the following hueristics:
     * -- If the database state is 'STATE_MODEL:', then update the
     *    'nodeMap.reverse' and 'nodeMap.map'
     *
     * -- If the database state is not STATE_MODEL, then leave the
     *    'nodeMap.reverse' and 'nodeMap.map' alone since they correspond to the
     *    information already written to the database. [May want to add a
     *    STATE_REDEFINE_MODEL]
     *
     * -- In both cases, update the nodeMap.reorder
     *
     * NOTE: The mapping is done on TRANSIENT fields only; MODEL fields
     *       should be in the orginal order...
     */
    if (!nodeMap.defined()) {
      if (nodeMap.map().empty()) {
        nodeMap.map().resize(num_to_get + 1);
        nodeMap.map()[0] = -1;
      }

      if (nodeMap.map()[0] == -1) {
        if (int_byte_size_api() == 4) {
          nodeMap.set_map(static_cast<int *>(ids), num_to_get, 0);
        }
        else {
          nodeMap.set_map(static_cast<int64_t *>(ids), num_to_get, 0);
        }
      }

      nodeMap.build_reverse_map();
      nodeMap.build_reorder_map(0, num_to_get);

      // Only a single nodeblock and all set
      assert(nodeMap.map()[0] == -1 || nodeMap.reverse().size() == (size_t)num_to_get);
      assert(get_region()->get_property("node_block_count").get_int() == 1);
      nodeMap.set_defined(true);
    }
    return num_to_get;
  }
}
