diff --git a/.gitattributes b/.gitattributes index 4468522e11a6b415755e3e7a00d6df84cde1db5c..b6e96457378deb097eda9f863c9bc0abd5e0f37f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,6 +27,7 @@ *.pvtu* filter=lfs diff=lfs merge=lfs -text *.gif filter=lfs diff=lfs merge=lfs -text *.mp4 filter=lfs diff=lfs merge=lfs -text +*.nc filter=lfs diff=lfs merge=lfs -text web/resources/_gen/**/*.content filter=lfs diff=lfs merge=lfs -text # Gocad files *.sg filter=lfs diff=lfs merge=lfs -text diff --git a/Applications/DataExplorer/NetCdfDialog/CMakeLists.txt b/Applications/DataExplorer/NetCdfDialog/CMakeLists.txt index 68c16fb9c323457236c350ec484c3d112dc96cab..a473593e20f2154fe5059c0236bc8ef9e5dedd23 100644 --- a/Applications/DataExplorer/NetCdfDialog/CMakeLists.txt +++ b/Applications/DataExplorer/NetCdfDialog/CMakeLists.txt @@ -5,14 +5,18 @@ if(BUILD_SHARED_LIBS) endif() target_link_libraries(NetCdfDialogLib PUBLIC Qt5::Widgets - PRIVATE MathLib vtknetcdfcpp) + ${NETCDF_LIBRARIES_CXX} + ${NETCDF_LIBRARIES_C} + ${HDF5_HL_LIBRARIES} + ${HDF5_LIBRARIES} + PRIVATE MathLib) set_property(TARGET NetCdfDialogLib PROPERTY FOLDER "DataExplorer") # Workaround for system installed VTK (tested on arch) if(NOT OGS_USE_CONAN) - target_include_directories( - NetCdfDialogLib SYSTEM - PUBLIC ${VTK_INSTALL_PREFIX}/include/vtk/vtknetcdfcpp) + target_include_directories(NetCdfDialogLib SYSTEM + PUBLIC ${NETCDF_INCLUDES_C} + PUBLIC ${NETCDF_INCLUDES_CXX}) endif() add_autogen_include(NetCdfDialogLib) diff --git a/Applications/DataExplorer/NetCdfDialog/NetCdfConfigure.ui b/Applications/DataExplorer/NetCdfDialog/NetCdfConfigure.ui index 6bcd090175f03a4424b5553a70ac9628cabffeeb..523cf2193ea1f80cc22e34ff31f6e8684fd1b0b3 100644 --- a/Applications/DataExplorer/NetCdfDialog/NetCdfConfigure.ui +++ b/Applications/DataExplorer/NetCdfDialog/NetCdfConfigure.ui @@ -167,68 +167,6 @@ </property> </widget> </item> - <item row="0" column="2"> - <widget class="QDateTimeEdit" name="dateTimeEditDim3"> - <property name="minimumSize"> - <size> - <width>110</width> - <height>0</height> - </size> - </property> - <property name="maximumSize"> - <size> - <width>110</width> - <height>16777215</height> - </size> - </property> - <property name="autoFillBackground"> - <bool>false</bool> - </property> - <property name="dateTime"> - <datetime> - <hour>0</hour> - <minute>0</minute> - <second>0</second> - <year>1900</year> - <month>1</month> - <day>1</day> - </datetime> - </property> - <property name="date"> - <date> - <year>1900</year> - <month>1</month> - <day>1</day> - </date> - </property> - <property name="maximumDateTime"> - <datetime> - <hour>23</hour> - <minute>59</minute> - <second>59</second> - <year>2200</year> - <month>12</month> - <day>31</day> - </datetime> - </property> - <property name="minimumDateTime"> - <datetime> - <hour>0</hour> - <minute>0</minute> - <second>0</second> - <year>1900</year> - <month>1</month> - <day>1</day> - </datetime> - </property> - <property name="displayFormat"> - <string>dd.MM.yyyy</string> - </property> - <property name="calendarPopup"> - <bool>true</bool> - </property> - </widget> - </item> <item row="3" column="3"> <spacer name="horizontalSpacer_8"> <property name="orientation"> @@ -255,6 +193,9 @@ </property> </spacer> </item> + <item row="0" column="2"> + <widget class="QSpinBox" name="dateTimeEditDim3"/> + </item> </layout> </widget> </item> @@ -720,7 +661,6 @@ <tabstop>doubleSpinBoxDim2Start</tabstop> <tabstop>doubleSpinBoxDim2End</tabstop> <tabstop>comboBoxDim3</tabstop> - <tabstop>dateTimeEditDim3</tabstop> <tabstop>comboBoxDim4</tabstop> <tabstop>spinBoxDim4</tabstop> <tabstop>doubleSpinBoxResolution</tabstop> diff --git a/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.cpp b/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.cpp index 23458e16ca4328437b4b4373f69f15dad7086585..610569cf022a46d1a06fc66d2b2ca04ba7e6ded3 100644 --- a/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.cpp +++ b/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.cpp @@ -21,16 +21,21 @@ #include <vtkImageImport.h> +using namespace netCDF; + // Constructor NetCdfConfigureDialog::NetCdfConfigureDialog(std::string const& fileName, QDialog* parent) -: QDialog(parent), _currentFile(new NcFile(fileName.c_str(), NcFile::ReadOnly)), - _currentInitialDateTime(QDateTime()), _currentMesh(nullptr), _currentRaster(nullptr), _currentPath(fileName) + : QDialog(parent), + _currentFile(fileName.c_str(), NcFile::read), + _currentMesh(nullptr), + _currentRaster(nullptr), + _currentPath(fileName) { setupUi(this); int const idx = setVariableSelect(); // set up variables of the file in the combobox comboBoxVariable->setCurrentIndex(idx); //pre-select the variable with the biggest number of dimensions...valueWithMaxDim() - _currentVar = _currentFile->get_var(idx); + _currentVar = _currentFile.getVar(comboBoxVariable->itemText(idx).toStdString()); setDimensionSelect(); @@ -45,7 +50,7 @@ NetCdfConfigureDialog::~NetCdfConfigureDialog() = default; void NetCdfConfigureDialog::accept() { QMessageBox valueErrorBox; - if (_currentVar->num_dims() < 2){ + if (_currentVar.getDimCount() < 2){ valueErrorBox.setText("Selected Variable has not enough dimensions."); valueErrorBox.exec(); }else if (doubleSpinBoxDim2Start->value() == doubleSpinBoxDim2Start->maximum()){ @@ -56,7 +61,6 @@ void NetCdfConfigureDialog::accept() valueErrorBox.exec(); }else{ createDataObject(); - delete _currentFile; this->done(QDialog::Accepted); } } @@ -64,13 +68,14 @@ void NetCdfConfigureDialog::accept() // Instructions if the Cancel-Button has been pressed. void NetCdfConfigureDialog::reject() { - delete _currentFile; this->done(QDialog::Rejected); } void NetCdfConfigureDialog::on_comboBoxVariable_currentIndexChanged(int id) { - _currentVar = _currentFile->get_var(_id_map[id]); + Q_UNUSED(id); + std::string const var_name = comboBoxVariable->currentText().toStdString(); + _currentVar = _currentFile.getVar(var_name); setDimensionSelect(); } @@ -80,7 +85,7 @@ void NetCdfConfigureDialog::on_comboBoxDim1_currentIndexChanged(int id) if (id == -1) id = 0; double firstValue=0, lastValue=0; unsigned size = 0; - getDimEdges(id,size,firstValue,lastValue); + getDimEdges(comboBoxDim1->currentText().toStdString(), size, firstValue, lastValue); doubleSpinBoxDim1Start->setValue(firstValue); doubleSpinBoxDim1End->setValue(lastValue); doubleSpinBoxResolution->setValue(getResolution()); @@ -89,12 +94,12 @@ void NetCdfConfigureDialog::on_comboBoxDim1_currentIndexChanged(int id) //set up y-axis/lon void NetCdfConfigureDialog::on_comboBoxDim2_currentIndexChanged(int id) { - if (_currentVar->num_dims() > 1) + if (_currentVar.getDimCount() > 1) { if (id == -1) id = 0; double firstValue=0, lastValue=0; unsigned size = 0; - getDimEdges(id,size,firstValue,lastValue); + getDimEdges(comboBoxDim2->currentText().toStdString(), size, firstValue, lastValue); doubleSpinBoxDim2Start->setValue(firstValue); doubleSpinBoxDim2End->setValue(lastValue); } @@ -103,37 +108,15 @@ void NetCdfConfigureDialog::on_comboBoxDim2_currentIndexChanged(int id) //set up time void NetCdfConfigureDialog::on_comboBoxDim3_currentIndexChanged(int id) { - if (_currentVar->num_dims() > 2) + if (_currentVar.getDimCount() > 2) { if (id == -1) id = 0; double firstValue=0, lastValue=0; unsigned size = 0; - getDimEdges(id,size,firstValue,lastValue); - - QTime firstTime(0,0,0), lastTime(0,0,0); - int firstDaysToAdd = 0, lastDaysToAdd = 0; - - getDaysTime(firstValue,firstTime,firstDaysToAdd); - getDaysTime(lastValue,lastTime,lastDaysToAdd); - - QDate initialDate(1960,1,1); - QTime initialTime(0,0); - - QDateTime initialDateTime; - initialDateTime.setDate(initialDate); - initialDateTime.setTime(initialTime); - - QDateTime firstDateTime = initialDateTime.addDays(firstDaysToAdd); - firstDateTime.setTime(firstTime); - - QDateTime lastDateTime = initialDateTime.addDays(lastDaysToAdd); - lastDateTime.setTime(lastTime); - - dateTimeEditDim3->setDateTime(firstDateTime); - dateTimeEditDim3->setMinimumDateTime(firstDateTime); - dateTimeEditDim3->setMaximumDateTime(lastDateTime); - - _currentInitialDateTime = initialDateTime; + getDimEdges(comboBoxDim3->currentText().toStdString(), size, firstValue, lastValue); + dateTimeEditDim3->setValue(static_cast<int>(firstValue)); + dateTimeEditDim3->setMinimum(static_cast<int>(firstValue)); + dateTimeEditDim3->setMaximum(static_cast<int>(lastValue)); lineEditName->setText(setName()); } } @@ -141,13 +124,12 @@ void NetCdfConfigureDialog::on_comboBoxDim3_currentIndexChanged(int id) //set up additional dimension void NetCdfConfigureDialog::on_comboBoxDim4_currentIndexChanged(int id) { - if (_currentVar->num_dims() > 3) + Q_UNUSED(id); + if (_currentVar.getDimCount() > 3) { - if (id == -1) id = 0; double firstValue=0, lastValue=0; unsigned size = 0; - getDimEdges(id,size,firstValue,lastValue); - // WARNING: Implicit conversion to int in spinBoxDim4->set*() + getDimEdges(comboBoxDim4->currentText().toStdString(), size, firstValue, lastValue); spinBoxDim4->setValue(static_cast<int>(firstValue)); spinBoxDim4->setMinimum(static_cast<int>(firstValue)); spinBoxDim4->setMaximum(static_cast<int>(lastValue)); @@ -158,17 +140,16 @@ int NetCdfConfigureDialog::setVariableSelect() { int max_dim = 0; int max_dim_idx = 0; - auto const n_vars = _currentFile->num_vars(); - for (int i = 0; i < n_vars; i++) + auto const& names =_currentFile.getVars(); + for (auto [name, var] : names) { - NcVar const& focusedVar = *_currentFile->get_var(i); - if (focusedVar.num_dims() > 1) + int const var_dim_count = var.getDimCount(); + if (var_dim_count > 1) { - _id_map.push_back(i); - comboBoxVariable->addItem(focusedVar.name()); - if (focusedVar.num_dims() > max_dim) + comboBoxVariable->addItem(QString::fromStdString(name)); + if (var_dim_count > max_dim) { - max_dim = focusedVar.num_dims(); + max_dim = var_dim_count; max_dim_idx = comboBoxVariable->count() - 1; } } @@ -178,36 +159,41 @@ int NetCdfConfigureDialog::setVariableSelect() void NetCdfConfigureDialog::setDimensionSelect() { - int const dim = _currentVar->num_dims(); + int const dim_count = _currentVar.getDimCount(); std::array<QComboBox*,4> dim_box = {{ comboBoxDim1, comboBoxDim2, comboBoxDim3, comboBoxDim4 }}; for (int i = 0; i < 4; ++i) { dim_box[i]->clear(); - dim_box[i]->setEnabled(i < dim); + dim_box[i]->setEnabled(i < dim_count); } - for (int i=0; i < dim; ++i) //write dimension-names into selection-boxes + // write dimension-names into selection-boxes + for (int i = 0; i < dim_count; ++i) { - for (int j = 0; j < dim; ++j) - dim_box[j]->addItem(_currentVar->get_dim(i)->name()); + for (int j = 0; j < dim_count; ++j) + { + dim_box[j]->addItem(QString::fromStdString(_currentVar.getDim(i).getName())); + } } - comboBoxDim1->setCurrentIndex(dim-2); - on_comboBoxDim1_currentIndexChanged(dim-2); - comboBoxDim2->setCurrentIndex(dim-1); - on_comboBoxDim2_currentIndexChanged(dim-1); - dateTimeEditDim3->setEnabled(dim > 2); // time is only enabled if dim > 2 - spinBoxDim4->setEnabled(dim > 3); // add. info is only enabled if dim > 3 - - if (dim > 2) + comboBoxDim1->setCurrentIndex(dim_count - 2); + on_comboBoxDim1_currentIndexChanged(dim_count - 2); + comboBoxDim2->setCurrentIndex(dim_count - 1); + on_comboBoxDim2_currentIndexChanged(dim_count - 1); + // time is only enabled if dim > 2 + dateTimeEditDim3->setEnabled(dim_count > 2); + // 3rd data dimension is only enabled if dim > 3 + spinBoxDim4->setEnabled(dim_count > 3); + + if (dim_count > 2) { comboBoxDim3->setCurrentIndex(0); on_comboBoxDim3_currentIndexChanged(0); } else - dateTimeEditDim3->setDateTime(_currentInitialDateTime); + dateTimeEditDim3->setSingleStep(0); - if (dim == 4) + if (dim_count == 4) { comboBoxDim4->setCurrentIndex(1); on_comboBoxDim4_currentIndexChanged(1); @@ -216,81 +202,49 @@ void NetCdfConfigureDialog::setDimensionSelect() spinBoxDim4->setValue(0); } -void NetCdfConfigureDialog::getDimEdges(int dimId, unsigned &size, double &firstValue, double &lastValue) +void NetCdfConfigureDialog::getDimEdges(std::string const& name, unsigned& size, + double& firstValue, double& lastValue) { - if ((_currentFile->get_var(_currentVar->get_dim(dimId)->name())) != nullptr) + size = 0; + firstValue = 0; + lastValue = 0; + if (_currentFile.getVar(name).isNull()) + return; + + NcVar const& tmpVarOfDim = _currentFile.getVar(name); + if ((tmpVarOfDim.getDimCount()) == 1) { - NcVar *tmpVarOfDim = _currentFile->get_var(_currentVar->get_dim(dimId)->name()); - if ((tmpVarOfDim->num_dims()) == 1) - { - int sizeOfDim = tmpVarOfDim->get_dim(0)->size(); - size = sizeOfDim; - double *arrayOfDimStart = new double[1]; //[1] = {0}; - long edgeOfArray = 1; //[1] = {1}; - long edgeOrigin[1] = {0}; - tmpVarOfDim->set_cur(edgeOrigin); - tmpVarOfDim->get(arrayOfDimStart,edgeOfArray); - firstValue = arrayOfDimStart[0]; - double arrayOfDimEnd[1] = {0}; - edgeOrigin[0] = sizeOfDim - 1; - tmpVarOfDim->set_cur(edgeOrigin); - tmpVarOfDim->get(arrayOfDimEnd,edgeOfArray); - lastValue = arrayOfDimEnd[0]; - } - }else{ - size = 0; - firstValue = 0; - lastValue = 0; + size = tmpVarOfDim.getDim(0).getSize(); + tmpVarOfDim.getVar({0}, {1}, &firstValue); + tmpVarOfDim.getVar({size - 1}, {1}, &lastValue); } } -void NetCdfConfigureDialog::getDaysTime(double minSince, QTime &time, int &days) -{ - auto tmpMin = (long)minSince; - long minuits = tmpMin % 60; - long tmpHours = tmpMin / 60; - long hours = tmpHours % 24; - days = (int)(tmpHours / 24); - time.setHMS(hours, minuits, 0); -} - -long NetCdfConfigureDialog::convertDateToMinutes(QDateTime initialDateTime, QDate selectedDate, QTime selectedTime) +int NetCdfConfigureDialog::getTimeStep() const { - long tmpInitialToSelectedDate = static_cast<long>(selectedDate.daysTo(initialDateTime.date())); - long selectedDays = - tmpInitialToSelectedDate * 24 * 60; - long selectedMinutes = (selectedTime.hour() * 60) + selectedTime.minute() + selectedDays; - return selectedMinutes; -} - -int NetCdfConfigureDialog::getTimeStep() -{ - NcVar* timeVar = _currentFile->get_var(comboBoxDim2->currentIndex()); - - double const datesToMinutes = convertDateToMinutes(_currentInitialDateTime,dateTimeEditDim3->date(),dateTimeEditDim3->time()); - - double timeArray[1] = {datesToMinutes}; - double currentTime = timeVar->get_index(timeArray); - if (currentTime < 0) currentTime=0; //if the value isn't found in the array, set it to 0 as default... - return static_cast<int>(currentTime); + return dateTimeEditDim3->value(); } int NetCdfConfigureDialog::getDim4() const { - NcVar* dim3Var = _currentFile->get_var(comboBoxDim4->currentIndex()); - int timeArray[1] = {spinBoxDim4->value()}; - int currentValueDim3 = dim3Var->get_index(timeArray); - if (currentValueDim3 < 0) currentValueDim3=0; //if the value isn't found in the array, set it to 0 as default... - return currentValueDim3; + NcVar const& dim3Var = + _currentFile.getVar(comboBoxDim4->currentText().toStdString()); + std::vector<std::size_t> start{static_cast<std::size_t>(spinBoxDim4->value())}; + int value(0); + dim3Var.getVar(start, {1}, &value); + if (value < 0) + value = 0; //if the value isn't found in the array, set it to 0 as default... + return value; } double NetCdfConfigureDialog::getResolution() { if (comboBoxDim1->currentIndex() > -1) { - NcVar* latVar = _currentFile->get_var(comboBoxDim1->currentIndex()); + NcVar const& var = _currentFile.getVar(comboBoxDim1->currentText().toStdString()); double firstValue = 0, lastValue = 0; unsigned size = 0; - getDimEdges(latVar->id(), size, firstValue, lastValue); + getDimEdges(var.getName(), size, firstValue, lastValue); if (size < 2) { return 1; @@ -306,43 +260,41 @@ double NetCdfConfigureDialog::getResolution() void NetCdfConfigureDialog::createDataObject() { - auto* length = new std::size_t[_currentVar->num_dims()]; double originLon = 0, originLat = 0; double lastLon = 0, lastLat = 0; unsigned sizeLon = 0, sizeLat = 0; - getDimEdges(comboBoxDim1->currentIndex(), sizeLat, originLat, lastLat); - getDimEdges(comboBoxDim2->currentIndex(), sizeLon, originLon, lastLon); - - - for(int i=0; i < _currentVar->num_dims(); i++) length[i]=1; - - // set array edges: lat x lon - length[comboBoxDim1->currentIndex()]=sizeLat; - length[comboBoxDim2->currentIndex()]=sizeLon; + std::string const dim1_name = comboBoxDim1->currentText().toStdString(); + getDimEdges(dim1_name, sizeLat, originLat, lastLat); + std::string const dim2_name = comboBoxDim2->currentText().toStdString(); + getDimEdges(dim2_name, sizeLon, originLon, lastLon); // set up array - auto* data_array = new double[sizeLat * sizeLon]; - for(std::size_t i=0; i < (sizeLat*sizeLon); i++) data_array[i]=0; + std::vector<double> data_array(sizeLat * sizeLon, 0); - //Time-Dimension: - if (_currentVar->num_dims() > 2) + std::vector<std::size_t> data_origin; + std::vector<std::size_t> data_length; + + if (_currentVar.getDimCount() > 2) { - auto* newOrigin = new long[_currentVar->num_dims()]; - for (int i=0; i < _currentVar->num_dims(); i++) newOrigin[i]=0; - newOrigin[comboBoxDim3->currentIndex()] = getTimeStep(); //set origin to selected time - _currentVar->set_cur(newOrigin); - //Dimension 4: - if (_currentVar->num_dims() > 3) newOrigin[comboBoxDim4->currentIndex()] = getDim4(); //if there are is a 4th dimension - delete [] newOrigin; + // time + data_origin.push_back(getTimeStep()); + data_length.push_back(1); + // 3rd dimension + if (_currentVar.getDimCount() > 3) + { + data_origin.push_back(getDim4()); + data_length.push_back(1); + } } - _currentVar->get(data_array,length); //create Array of Values + data_origin.push_back(0); // x-origin + data_origin.push_back(0); // y-origin + data_length.push_back(sizeLat); + data_length.push_back(sizeLon); + _currentVar.getVar(data_origin, data_length, data_array.data()); - for (std::size_t i=0; i < (sizeLat*sizeLon); i++) - { - //data_array[i] = data_array[i] - 273; // convert from kalvin to celsius - if (data_array[i] < -9999 ) data_array[i] = -9999; // all values < -10000, set to "no-value" - } + std::replace_if(data_array.begin(), data_array.end(), + [](double& x) { return x <= -9999; }, -9999); double origin_x = (originLon < lastLon) ? originLon : lastLon; double origin_y = (originLat < lastLat) ? originLat : lastLat; @@ -350,7 +302,7 @@ void NetCdfConfigureDialog::createDataObject() double resolution = (doubleSpinBoxResolution->value()); if (originLat > lastLat) // reverse lines in vertical direction if the original file has its origin in the northwest corner - this->reverseNorthSouth(data_array, sizeLon, sizeLat); + this->reverseNorthSouth(data_array.data(), sizeLon, sizeLat); GeoLib::RasterHeader const header = {sizeLon, sizeLat, 1, origin, resolution, -9999}; @@ -371,17 +323,14 @@ void NetCdfConfigureDialog::createDataObject() useIntensity = MeshLib::UseIntensityAs::DATAVECTOR; } _currentMesh = MeshLib::RasterToMesh::convert( - data_array, header, meshElemType, useIntensity, _currentVar->name()); + data_array.data(), header, meshElemType, useIntensity, _currentVar.getName()); } else { - vtkImageImport* image = VtkRaster::loadImageFromArray(data_array, header); + vtkImageImport* image = VtkRaster::loadImageFromArray(data_array.data(), header); _currentRaster = VtkGeoImageSource::New(); _currentRaster->setImage(image, QString::fromStdString(this->getName()), origin[0], origin[1], resolution); } - - delete[] length; - delete[] data_array; } QString NetCdfConfigureDialog::setName() @@ -396,7 +345,7 @@ QString NetCdfConfigureDialog::setName() std::string NetCdfConfigureDialog::getName() { std::string name = (lineEditName->text()).toStdString(); - QString date = dateTimeEditDim3->date().toString(Qt::LocalDate); + QString const date = QString::number(dateTimeEditDim3->value()); name.append(" - ").append(date.toStdString()); return name; } @@ -439,5 +388,3 @@ void NetCdfConfigureDialog::on_radioMesh_toggled(bool isTrue) this->comboBoxUseIntensity->setEnabled(true); } } - - diff --git a/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.h b/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.h index eb23e363acdad4b2a823cc88c33dee7e4bbae0a7..5a25c14109700806bba7002a5c60a90e1b206593 100644 --- a/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.h +++ b/Applications/DataExplorer/NetCdfDialog/NetCdfConfigureDialog.h @@ -9,7 +9,7 @@ #pragma once -#include <netcdfcpp.h> +#include <netcdf> #include <QDialog> #include "ui_NetCdfConfigure.h" @@ -55,21 +55,20 @@ private: /// returns the index of the first variable with the highest dimension. int setVariableSelect(); void setDimensionSelect(); - void getDimEdges(int dimId, unsigned &size, double &firstValue, double &lastValue); - void getDaysTime(double minSince, QTime &time, int &days); - long convertDateToMinutes(QDateTime initialDateTime,QDate selectedDate, QTime selectedTime); + void getDimEdges(std::string const& name, + unsigned& size, + double& firstValue, + double& lastValue); void createDataObject(); - int getTimeStep(); + int getTimeStep() const; int getDim4() const; double getResolution(); QString setName(); void reverseNorthSouth(double* data, std::size_t width, std::size_t height); - NcFile *_currentFile; - NcVar *_currentVar; - QDateTime _currentInitialDateTime; + netCDF::NcFile _currentFile; + netCDF::NcVar _currentVar; MeshLib::Mesh* _currentMesh; VtkGeoImageSource* _currentRaster; std::string _currentPath; - std::vector<int> _id_map; }; diff --git a/Applications/Utils/FileConverter/CMakeLists.txt b/Applications/Utils/FileConverter/CMakeLists.txt index 761d8a6e8f8352e372aad047e94b1b16860a5298..28882437a085b8131fe26d1786240fbcb41b3acb 100644 --- a/Applications/Utils/FileConverter/CMakeLists.txt +++ b/Applications/Utils/FileConverter/CMakeLists.txt @@ -11,6 +11,10 @@ set(TOOLS GocadTSurfaceReader Mesh2Raster) +if(OGS_USE_NETCDF) + list(APPEND TOOLS NetCdfConverter) +endif() + if(OGS_BUILD_GUI) if(Shapelib_FOUND) list(APPEND TOOLS ConvertSHPToGLI) @@ -37,3 +41,10 @@ if(TARGET Mesh2Shape) target_link_libraries(Mesh2Shape ${Shapelib_LIBRARIES}) endif() +if(TARGET NetCdfConverter) + target_link_libraries(NetCdfConverter + ${NETCDF_LIBRARIES_CXX} + ${NETCDF_LIBRARIES_C} + ${HDF5_HL_LIBRARIES} + ${HDF5_LIBRARIES}) +endif() diff --git a/Applications/Utils/FileConverter/NetCdfConverter.cpp b/Applications/Utils/FileConverter/NetCdfConverter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..95b0f4b14ee87912370b1bd23cc20637b14052ee --- /dev/null +++ b/Applications/Utils/FileConverter/NetCdfConverter.cpp @@ -0,0 +1,726 @@ +/** + * \file + * @copyright + * Copyright (c) 2012-2019, OpenGeoSys Community (http://www.opengeosys.org) + * Distributed under a Modified BSD License. + * See accompanying file LICENSE.txt or + * http://www.opengeosys.org/LICENSE.txt + */ + +// STL +#include <cctype> +#include <iostream> +#include <limits> +#include <memory> +#include <numeric> +#include <sstream> +#include <string> +#include <utility> + +#include <tclap/CmdLine.h> + +#include "Applications/ApplicationsLib/LogogSetup.h" + +#include <netcdf> + +#include "BaseLib/FileTools.h" +#include "BaseLib/LogogSimpleFormatter.h" +#include "InfoLib/GitInfo.h" +#include "GeoLib/Raster.h" +#include "MeshLib/Mesh.h" +#include "MeshLib/MeshGenerators/RasterToMesh.h" +#include "MeshLib/IO/VtkIO/VtuInterface.h" + +using namespace netCDF; + +const double no_data = -9999; + +static void checkExit(std::string const& str) +{ + if (str == "x" || str == "exit") + exit(0); +} + +static void showErrorMessage(std::size_t const error_id, + std::size_t const max = 0) +{ + if (error_id == 0) + { + ERR("Input not valid."); + } + else if (error_id == 1) + { + ERR("Index not valid. Valid indices are in [0,%d].", max); + } + else if (error_id == 2) + { + ERR("Input not valid."); + std::cout << "Type \"info\" to display the available options again. " + "\"exit\" will exit the programme.\n"; + } +} + +static std::size_t parseInput(std::string const& request_str, + std::size_t const max_val, + bool const has_info = false) +{ + while (true) + { + std::cout << request_str; + std::string str; + std::size_t val; + std::getline(std::cin, str); + checkExit(str); + if (has_info && str == "info") + return (max_val); + std::stringstream str_stream(str); + if (!(str_stream >> val)) + { + std::size_t const error_val = (has_info) ? 2 : 0; + showErrorMessage(error_val); + continue; + } + if (val > max_val - 1) + { + showErrorMessage(1, max_val - 1); + continue; + } + return val; + } + return std::numeric_limits<std::size_t>::max(); +} + +static NcVar getDimVar(NcFile const& dataset, NcVar const& var, + std::size_t const dim) +{ + NcDim const& dim_obj = var.getDim(dim); + return dataset.getVar(dim_obj.getName()); +} + +static std::pair<double, double> getDimLength(NcVar const& var, + std::size_t const dim) +{ + return std::make_pair(0.0, static_cast<double>(var.getDim(dim).getSize())); +} + +static std::vector<std::string> getArrays(NcFile const& dataset) +{ + auto const& names = dataset.getVars(); + std::vector<std::string> var_names; + for (auto [name, var] : names) + { + (void)var; + var_names.push_back(name); + } + return var_names; +} + +static void showArrays(NcFile const& dataset) +{ + std::size_t const n_vars(dataset.getDimCount()); + std::cout << "The NetCDF file contains the following " << n_vars + << " arrays:\n\n"; + std::cout << "\tIndex\tArray Name\t#Dimensions\n"; + std::cout << "-------------------------------------------\n"; + auto const& names = dataset.getVars(); + std::size_t count = 0; + for (auto [name, var] : names) + { + std::cout << "\t" << count++ << "\t" << name << "\t(" + << var.getDimCount() << "D array)\n"; + } + std::cout << "\n"; +} + +static void showArraysDims(NcVar const& var) +{ + std::cout << "Data array \"" << var.getName() + << "\" contains the following dimensions:\n"; + std::size_t const n_dims(var.getDimCount()); + for (std::size_t i = 0; i < n_dims; ++i) + std::cout << "\t" << i << "\t" << var.getDim(i).getName() << "\t(" + << var.getDim(i).getSize() << " values)\n"; + std::cout << "\n"; +} + +static std::pair<double, double> getBoundaries(NcVar const& var) +{ + if (var.getDimCount() == 1) + { + double start, end; + std::size_t const size = var.getDim(0).getSize(); + var.getVar({0}, {1}, &start); + var.getVar({size - 1}, {1}, &end); + return std::make_pair(start, end); + } + return std::make_pair(0, 0); +} + +static MathLib::Point3d getOrigin(NcFile const& dataset, NcVar const& var, + std::vector<std::size_t> const& dim_idx_map, + bool is_time_dep) +{ + std::size_t const temp_offset = (is_time_dep) ? 1 : 0; + MathLib::Point3d origin(MathLib::ORIGIN); + std::size_t const n_dims = var.getDimCount(); + for (std::size_t i = temp_offset; i < n_dims; ++i) + { + NcVar const& dim = getDimVar(dataset, var, dim_idx_map[i]); + auto const bounds = (dim.isNull()) ? getDimLength(var, dim_idx_map[i]) + : getBoundaries(dim); + origin[i - temp_offset] = + (bounds.first < bounds.second) ? bounds.first : bounds.second; + } + return origin; +} + +static void flipRaster(std::vector<double>& data, std::size_t const width, + std::size_t const height) +{ + std::size_t const length(data.size()); + std::vector<double> tmp_vec(length); + for (std::size_t i = 0; i < height; i++) + { + std::size_t const line_idx(length - (width * (i + 1))); + for (std::size_t j = 0; j < width; j++) + { + tmp_vec.push_back(data[line_idx + j]); + } + } + std::copy(tmp_vec.cbegin(), tmp_vec.cend(), data.begin()); +} + +static bool canConvert(NcVar const& var) +{ + bool ret(var.getDimCount() < 2); + if (ret) + ERR("Only 2+ dimensional variables can be converted into OGS Meshes.\n"); + return !ret; +} + +static std::string arraySelectionLoop(NcFile const& dataset) +{ + std::vector<std::string> const& names = getArrays(dataset); + showArrays(dataset); + std::size_t const idx = + parseInput("Enter data array index: ", dataset.getVarCount(), true); + + if (static_cast<int>(idx) == dataset.getVarCount() || + !canConvert(dataset.getVar(names[idx]))) + return arraySelectionLoop(dataset); + + return names[idx]; +} + +static bool dimensionSelectionLoop(NcVar const& var, + std::vector<std::size_t>& dim_idx_map) +{ + showArraysDims(var); + std::size_t const n_dims(var.getDimCount()); + dim_idx_map[0] = std::numeric_limits<std::size_t>::max(); + bool is_time_dep(true); + + // get temporal dimension + if (n_dims > 1) + { + std::string temp_str(""); + cout << "Is the parameter time-dependent?\n"; + while (dim_idx_map[0] == std::numeric_limits<std::size_t>::max() && + is_time_dep == true) + { + cout << "Enter ID for temporal dimension or \"c\" to continue: "; + std::getline(std::cin, temp_str); + std::stringstream str_stream(temp_str); + if (str_stream.str() == "c" || str_stream.str() == "continue") + is_time_dep = false; + else + { + if (!(str_stream >> dim_idx_map[0])) + { + showErrorMessage(0); + dim_idx_map[0] = std::numeric_limits<std::size_t>::max(); + continue; + } + if (dim_idx_map[0] > n_dims - 1) + { + showErrorMessage(1, var.getDimCount() - 1); + dim_idx_map[0] = std::numeric_limits<std::size_t>::max(); + } + } + } + } + else + is_time_dep = false; + + // get spatial dimension(s) + std::size_t const start_idx = (is_time_dep) ? 1 : 0; + std::array<std::string, 4> const dim_comment{ + "(x / longitude)", "(y / latitude)", "(z / height / depth)", + "[Error: 4-dimensional non-temporal arrays are not supported]"}; + for (std::size_t i = start_idx; i < n_dims; ++i) + { + dim_idx_map[i] = std::numeric_limits<std::size_t>::max(); + + std::string const request_str("Enter ID for dimension " + std::to_string(i) + + " " + dim_comment[i - start_idx] + ": "); + std::size_t const idx = parseInput(request_str, var.getDimCount(), true); + + if (static_cast<int>(idx) == var.getDimCount()) + { + showArraysDims(var); + i--; + continue; + } + dim_idx_map[i] = idx; + } + + return is_time_dep; +} + +static std::pair<std::size_t, std::size_t> timestepSelectionLoop( + NcVar const& var, std::size_t const dim_idx) +{ + std::size_t const n_time_steps = var.getDim(dim_idx).getSize(); + std::size_t const max_val = std::numeric_limits<std::size_t>::max(); + std::pair<std::size_t, std::size_t> bounds(max_val, max_val); + std::cout << "\nThe dataset contains " << n_time_steps << " time steps.\n"; + while (bounds.first == max_val) + { + bounds.first = parseInput("Specify first time step to export: ", n_time_steps, false); + } + while (bounds.first > bounds.second || bounds.second > n_time_steps) + { + bounds.second = parseInput( "Specify last time step to export: ", n_time_steps, false); + } + return bounds; +} + +static MeshLib::MeshElemType elemSelectionLoop(std::size_t const dim) +{ + if (dim == 1) + return MeshLib::MeshElemType::LINE; + + MeshLib::MeshElemType t = MeshLib::MeshElemType::INVALID; + while (t == MeshLib::MeshElemType::INVALID) + { + std::cout << "\nSelect element type for result, choose "; + + if (dim == 2) + std::cout << "(t)riangle or (q)uadliteral: "; + if (dim == 3) + std::cout << "(p)rism or (h)exahedron: "; + std::string type(""); + std::getline(std::cin, type); + checkExit(type); + if (dim == 2) + { + if (type != "t" && type != "q" && type != "tri" && type != "quad" && + type != "triangle" && type != "quatliteral") + continue; + if (type == "t" || type == "tri" || type == "triangle") + return MeshLib::MeshElemType::TRIANGLE; + return MeshLib::MeshElemType::QUAD; + } + + if (dim == 3) + { + if (type != "p" && type != "h" && type != "prism" && + type != "hex" && type != "hexahedron") + continue; + if (type == "p" || type == "prism") + return MeshLib::MeshElemType::PRISM; + return MeshLib::MeshElemType::HEXAHEDRON; + } + } + return t; +} + +static bool multFilesSelectionLoop( + std::pair<std::size_t, std::size_t> const& time_bounds) +{ + std::size_t const n_time_steps(time_bounds.second - time_bounds.first + 1); + std::cout << "\nThe selection includes " << n_time_steps + << " time steps.\n"; + std::cout << "0. Save data in " << n_time_steps + << " mesh files with one scalar array each.\n"; + std::cout << "1. Save data in one mesh file with " << n_time_steps + << " scalar arrays.\n"; + std::size_t ret = parseInput("Select preferred method: ", 2, false); + return (ret != 0); +} + +static std::string getIterationString(std::size_t i, std::size_t max) +{ + std::size_t const max_length(std::to_string(max).length()); + std::string const current_str(std::to_string(i)); + return std::string(max_length - current_str.length(), '0') + current_str; +} + +static double getResolution(NcFile const& dataset, NcVar const& var) +{ + std::size_t const dim_idx = var.getDimCount() - 1; + NcVar const dim_var (getDimVar(dataset, var, dim_idx)); + auto const bounds = (dim_var.isNull()) ? getDimLength(var, dim_idx) + : getBoundaries(dim_var); + std::size_t const dim_size = var.getDim(dim_idx).getSize(); + std::string const err_msg("Dimension \"" + var.getDim(dim_idx).getName() + + "\" has size 0. Aborting..."); + if (dim_size == 0) return OGS_FATAL(err_msg.c_str()); + return std::fabs(bounds.second - bounds.first) / static_cast<double>(dim_size); +} + +static GeoLib::RasterHeader createRasterHeader( + NcFile const& dataset, NcVar const& var, + std::vector<std::size_t> const& dim_idx_map, + std::vector<std::size_t> const& length, bool is_time_dep) +{ + MathLib::Point3d const origin = getOrigin(dataset, var, dim_idx_map, is_time_dep); + double const res = getResolution(dataset, var); + std::size_t n_dims = var.getDimCount(); + std::size_t temp_offset = (is_time_dep) ? 1 : 0; + std::size_t z_length = + (n_dims - temp_offset == 3) ? length[dim_idx_map.back()] : 1; + return {length[dim_idx_map[0 + temp_offset]], + length[dim_idx_map[1 + temp_offset]], + z_length, origin, res, no_data}; +} + +static std::vector<std::size_t> getLength(NcVar const& var, bool const is_time_dep) +{ + std::size_t const temp_offset = (is_time_dep) ? 1 : 0; + std::size_t const n_dims = (var.getDimCount()); + std::vector<std::size_t> length(n_dims, 1); + for (std::size_t i = temp_offset; i < n_dims; ++i) + { + length[i] = var.getDim(i).getSize(); + } + return length; +} + +static std::vector<double> getData(NcFile const& dataset, NcVar const& var, + std::size_t const total_length, + std::size_t const time_step, + std::vector<std::size_t> const& length) +{ + std::size_t const n_dims(var.getDimCount()); + std::vector<std::size_t> offset(n_dims, 0); + offset[0] = time_step; + std::vector<double> data_vec(total_length, 0); + var.getVar(offset, length, data_vec.data()); + std::replace_if(data_vec.begin(), data_vec.end(), + [](double& x) { return x <= no_data; }, no_data); + + // reverse lines in vertical direction if the original file has its origin + // in the northwest corner + NcVar const dim_var (getDimVar(dataset, var, n_dims - 1)); + auto const bounds = (dim_var.isNull()) ? getDimLength(var, n_dims - 1) + : getBoundaries(dim_var); + if (bounds.first > bounds.second) + flipRaster(data_vec, length[n_dims - 2], length[n_dims - 1]); + return data_vec; +} + +static bool assignDimParams(NcVar const& var, + std::vector<std::size_t>& dim_idx_map, + TCLAP::ValueArg<std::size_t>& arg_dim_time, + TCLAP::ValueArg<std::size_t>& arg_dim1, + TCLAP::ValueArg<std::size_t>& arg_dim2, + TCLAP::ValueArg<std::size_t>& arg_dim3) +{ + std::size_t dim_param_count = 0; + if (arg_dim_time.isSet()) + dim_param_count++; + if (arg_dim1.isSet()) + dim_param_count++; + if (arg_dim2.isSet()) + dim_param_count++; + if (arg_dim3.isSet()) + dim_param_count++; + + std::size_t const n_dims = var.getDimCount(); + if (dim_param_count != n_dims) + { + ERR("Number of parameters set does not fit number of parameters for specified variable."); + return false; + } + + if (arg_dim_time.getValue() >= n_dims || arg_dim1.getValue() >= n_dims || + arg_dim2.getValue() >= n_dims || arg_dim3.getValue() >= n_dims) + { + ERR("Maximum allowed dimension for variable \"%s\" is %d.", var.getName().c_str(), n_dims-1); + return false; + } + + if (arg_dim_time.isSet()) + dim_idx_map[0] = arg_dim_time.getValue(); + std::size_t const temp_offset = (arg_dim_time.isSet()) ? 1 : 0; + dim_idx_map[0 + temp_offset] = arg_dim1.getValue(); + dim_idx_map[1 + temp_offset] = arg_dim2.getValue(); + if (n_dims == (3 + temp_offset)) + dim_idx_map[2 + temp_offset] = arg_dim3.getValue(); + + return true; +} + +static std::pair<std::size_t, std::size_t> assignTimeBounds( + NcVar const& var, + TCLAP::ValueArg<std::size_t>& arg_time_start, + TCLAP::ValueArg<std::size_t>& arg_time_end) +{ + auto const bounds = getBoundaries(var); + if (arg_time_start.getValue() > bounds.second) + { + ERR("Start time step larger than total number of time steps. Resetting to 0."); + arg_time_start.reset(); + } + + if (!arg_time_end.isSet()) + return {arg_time_start.getValue(), arg_time_start.getValue()}; + + if (arg_time_end.getValue() > bounds.second) + { + ERR("End time step larger than total number of time steps. Resetting to starting time step"); + return {arg_time_start.getValue(), arg_time_start.getValue()}; + } + + if (arg_time_end.getValue() < arg_time_start.getValue()) + { + ERR("End time step larger than starting time step. Swapping values"); + return {arg_time_end.getValue(), arg_time_start.getValue()}; + } + + return {arg_time_start.getValue(), arg_time_end.getValue()}; +} + +static MeshLib::MeshElemType assignElemType( + TCLAP::ValueArg<std::string>& arg_elem_type) +{ + if (arg_elem_type.getValue() == "tri") + return MeshLib::MeshElemType::TRIANGLE; + if (arg_elem_type.getValue() == "quad") + return MeshLib::MeshElemType::QUAD; + if (arg_elem_type.getValue() == "prism") + return MeshLib::MeshElemType::PRISM; + if (arg_elem_type.getValue() == "hex") + return MeshLib::MeshElemType::HEXAHEDRON; + // this should never happen + return MeshLib::MeshElemType::INVALID; +} + +static bool convert(NcFile const& dataset, NcVar const& var, + std::string const& output_name, + std::vector<std::size_t> const& dim_idx_map, + bool const is_time_dep, + std::pair<std::size_t, std::size_t> const& time_bounds, + bool const use_single_file, MeshLib::MeshElemType const elem_type) +{ + std::unique_ptr<MeshLib::Mesh> mesh; + std::vector<std::size_t> const length = getLength(var, is_time_dep); + std::size_t const array_length = std::accumulate( + length.cbegin(), length.cend(), 1, std::multiplies<std::size_t>()); + for (std::size_t i = time_bounds.first; i <= time_bounds.second; ++i) + { + std::string const step_str = (time_bounds.first != time_bounds.second) + ? std::string(" time step " + std::to_string(i)) : ""; + std::cout << "Converting" << step_str << "...\n"; + std::vector<double> const data_vec = + getData(dataset, var, array_length, i, length); + + GeoLib::RasterHeader const header = + createRasterHeader(dataset, var, dim_idx_map, length, is_time_dep); + MeshLib::UseIntensityAs const useIntensity = + MeshLib::UseIntensityAs::DATAVECTOR; + if (!use_single_file) + { + mesh.reset(MeshLib::RasterToMesh::convert( + data_vec.data(), header, elem_type, useIntensity, + var.getName())); + std::string const output_file_name( + BaseLib::dropFileExtension(output_name) + + getIterationString(i, time_bounds.second) + ".vtu"); + MeshLib::IO::VtuInterface vtu(mesh.get()); + vtu.writeToFile(output_file_name); + } + else + { + std::string array_name(var.getName()); + if (time_bounds.first != time_bounds.second) + array_name.append(getIterationString(i, time_bounds.second)); + if (i == time_bounds.first) // create persistent mesh + mesh.reset(MeshLib::RasterToMesh::convert( + data_vec.data(), header, elem_type, useIntensity, + array_name)); + else // copy array to mesh + { + std::unique_ptr<MeshLib::Mesh> const temp( + MeshLib::RasterToMesh::convert(data_vec.data(), header, + elem_type, useIntensity, + array_name)); + MeshLib::PropertyVector<double> const* const vec = + temp->getProperties().getPropertyVector<double>(array_name); + if (vec == nullptr) + return false; + MeshLib::addPropertyToMesh<double>( + *mesh, array_name, MeshLib::MeshItemType::Cell, 1, *vec); + } + if (i == time_bounds.second) + { + MeshLib::IO::VtuInterface vtu(mesh.get()); + vtu.writeToFile(output_name); + } + } + } + return true; +} + +int main(int argc, char* argv[]) +{ + ApplicationsLib::LogogSetup logog_setup; + + TCLAP::CmdLine cmd( + "Converts NetCDF data into mesh file(s).\n\n " + "OpenGeoSys-6 software, version " + + GitInfoLib::GitInfo::ogs_version + + ".\n" + "Copyright (c) 2012-2019, OpenGeoSys Community " + "(http://www.opengeosys.org)", + ' ', GitInfoLib::GitInfo::ogs_version); + + std::vector<std::string> allowed_elems{"tri", "quad", "prism", "hex"}; + TCLAP::ValuesConstraint<std::string> allowed_elem_vals(allowed_elems); + TCLAP::ValueArg<std::string> arg_elem_type( + "e", "elem-type", "the element type used in the resulting OGS mesh", + false, "", &allowed_elem_vals); + cmd.add(arg_elem_type); + + TCLAP::SwitchArg arg_single_file( + "", "single-file", + "if set, all time steps will be written to a single mesh file (with " + "one scalar array per time step)"); + cmd.add(arg_single_file); + + TCLAP::ValueArg<std::size_t> arg_time_end( + "", "timestep-last", + "last time step to be extracted (only for time-dependent variables!)", + false, 0, "integer specifying index of time step"); + cmd.add(arg_time_end); + + TCLAP::ValueArg<std::size_t> arg_time_start( + "", "timestep-first", + "first time step to be extracted (only for time-dependent variables!)", + false, 0, "integer specifying index of time step"); + cmd.add(arg_time_start); + + std::vector<std::size_t> allowed_dims{0, 1, 2, 3}; + TCLAP::ValuesConstraint<std::size_t> allowed_dim_vals(allowed_dims); + TCLAP::ValueArg<std::size_t> arg_dim3( + "", "dim3", + "index of third dimension (z/height/depth) for the selected variable", + false, 0, &allowed_dim_vals); + cmd.add(arg_dim3); + + TCLAP::ValueArg<std::size_t> arg_dim2( + "", "dim2", + "index of second dimension (y/latitude) for the selected " + "variable", + false, 0, &allowed_dim_vals); + cmd.add(arg_dim2); + + TCLAP::ValueArg<std::size_t> arg_dim1( + "", "dim1", + "index of first dimension (x/longitude) for the selected variable", + false, 0, &allowed_dim_vals); + cmd.add(arg_dim1); + + TCLAP::ValueArg<std::size_t> arg_dim_time( + "t", "time", + "index of the time-dependent dimension for the selected variable", + false, 0, &allowed_dim_vals); + cmd.add(arg_dim_time); + + TCLAP::ValueArg<std::string> arg_varname( + "v", "var", "variable included in the the netCDF file", false, "", + "string containing the variable name"); + cmd.add(arg_varname); + + TCLAP::ValueArg<std::string> arg_output( + "o", "output", "the OGS mesh output file", true, "", + "string containing the path and file name"); + cmd.add(arg_output); + TCLAP::ValueArg<std::string> arg_input( + "i", "input", "the netCDF input file", true, "", + "string containing the path and file name"); + cmd.add(arg_input); + cmd.parse(argc, argv); + + NcFile dataset(arg_input.getValue().c_str(), NcFile::read); + + if (dataset.isNull()) + { + ERR("Error opening file."); + return -1; + } + + std::cout << "OpenGeoSys NetCDF Converter\n"; + if (!arg_varname.isSet()) + { + std::cout << "File " << arg_input.getValue() + << " loaded. Press ENTER to display available data arrays.\n"; + std::cin.ignore(); + } + + std::string const& output_name(arg_output.getValue()); + std::string const& var_name = (arg_varname.isSet()) + ? arg_varname.getValue() + : arraySelectionLoop(dataset); + NcVar const& var = dataset.getVar(var_name); + if (var.isNull()) + { + ERR("Variable \"%s\" not found in file.", arg_varname.getValue().c_str()); + return EXIT_FAILURE; + } + + std::vector<std::size_t> dim_idx_map(var.getDimCount(), 0); + bool is_time_dep (false); + if (arg_dim1.isSet() && arg_dim2.isSet()) + { + is_time_dep = arg_dim_time.isSet(); + if (!assignDimParams(var, dim_idx_map, arg_dim_time, arg_dim1, arg_dim2, arg_dim3)) + return EXIT_FAILURE; + } + else + { + is_time_dep = dimensionSelectionLoop(var, dim_idx_map); + } + + std::pair<std::size_t, std::size_t> time_bounds(0, 0); + if (is_time_dep) + time_bounds = (arg_time_start.isSet()) + ? assignTimeBounds(getDimVar(dataset, var, dim_idx_map[0]), + arg_time_start, arg_time_end) + : timestepSelectionLoop(var, dim_idx_map[0]); + + bool use_single_file(true); + if (arg_time_start.isSet() && time_bounds.first != time_bounds.second) + { + use_single_file = arg_single_file.isSet(); + } + else + { + if (is_time_dep && time_bounds.first != time_bounds.second) + use_single_file = multFilesSelectionLoop(time_bounds); + } + + std::size_t const temp_offset = (is_time_dep) ? 1 : 0; + std::size_t const n_dims = (var.getDimCount()); + MeshLib::MeshElemType const elem_type = (arg_elem_type.isSet()) + ? assignElemType(arg_elem_type) + : elemSelectionLoop(n_dims - temp_offset); + + if (!convert(dataset, var, output_name, dim_idx_map, is_time_dep, + time_bounds, use_single_file, elem_type)) + return EXIT_FAILURE; + + std::cout << "Conversion finished successfully.\n"; + return EXIT_SUCCESS; +} diff --git a/Applications/Utils/Tests.cmake b/Applications/Utils/Tests.cmake index 6fffe0a5ed3f1d529abea405ff4131b99f866778..95de6c8b7184c286660d1de87326f5c4923d4bff 100644 --- a/Applications/Utils/Tests.cmake +++ b/Applications/Utils/Tests.cmake @@ -285,3 +285,27 @@ AddTest( DIFF_DATA AmmerSubsurfaceGrid.vtu AmmerGridOutput.vtu MaterialIDs MaterialIDs 0 0 ) + +if(OGS_USE_NETCDF) + AddTest( + NAME NetCDF_2D_Test + PATH FileConverter/ + EXECUTABLE NetCdfConverter + EXECUTABLE_ARGS -i sresa1b_ncar_ccsm3-example.nc -o ${Data_BINARY_DIR}/FileConverter/sresa1b_ncar_ccsm3-example.vtu -v pr -t 0 --dim1 2 --dim2 1 --timestep-first 0 --timestep-last 0 -e tri + REQUIREMENTS NOT OGS_USE_MPI + TESTER vtkdiff + DIFF_DATA + sresa1b_ncar_ccsm3-example.vtu sresa1b_ncar_ccsm3-example.vtu pr pr 1e-16 0 + ) + + AddTest( + NAME NetCDF_3D_Test + PATH FileConverter/ + EXECUTABLE NetCdfConverter + EXECUTABLE_ARGS -i slim_100897_198.nc -o ${Data_BINARY_DIR}/FileConverter/slim_100897_198.vtu -v NO -t 0 --dim1 3 --dim2 2 --dim3 1 --timestep-first 0 --timestep-last 0 -e hex + REQUIREMENTS NOT OGS_USE_MPI + TESTER vtkdiff + DIFF_DATA + slim_100897_198.vtu slim_100897_198.vtu NO NO 1e-16 0 + ) +endif() diff --git a/Jenkinsfile b/Jenkinsfile index 5436975688a6815a77e5675f822c6ff7cba10945..30d3d9b93ad188f0b7cba8479136d460ad1fd2f4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -204,7 +204,8 @@ pipeline { '-DOGS_BUILD_GUI=ON ' + '-DOGS_BUILD_UTILS=ON ' + '-DOGS_CONAN_BUILD=missing ' + - '-DOGS_BUILD_TESTS=OFF ' + '-DOGS_BUILD_TESTS=OFF ' + + '-DOGS_USE_NETCDF=ON ' } build { target="package" @@ -225,6 +226,7 @@ pipeline { always { recordIssues enabledForFailure: true, filters: [ excludeFile('.*qrc_icons\\.cpp.*'), + excludeFile('ncGroup\\.h'), excludeMessage('.*tmpnam.*')], tools: [gcc4(name: 'GCC-GUI', id: 'gcc4-gui', pattern: 'build/build*.log')], @@ -232,7 +234,7 @@ pipeline { recordIssues enabledForFailure: true, tools: [cppCheck(pattern: 'build/cppcheck.log')] } - success { archiveArtifacts 'build/*.tar.gz,build/conaninfo.txt' } + success { archiveArtifacts 'build/*.tar.gz,build/conaninfo.txt,build/cppcheck.log' } } } // ********************* Docker-Conan-Debug **************************** @@ -406,7 +408,8 @@ pipeline { '-DOGS_BUILD_GUI=ON ' + '-DOGS_BUILD_UTILS=ON ' + '-DOGS_CONAN_BUILD=missing ' + - '-DOGS_BUILD_SWMM=ON ' + '-DOGS_BUILD_SWMM=ON ' + + '-DOGS_USE_NETCDF=ON ' } build { target="package" @@ -460,10 +463,10 @@ pipeline { "-DBUILD_SHARED_LIBS=${build_shared} " + '-DOGS_CPU_ARCHITECTURE=core2 ' + '-DOGS_DOWNLOAD_ADDITIONAL_CONTENT=ON ' + - '-DOGS_BUILD_GUI=OFF ' + // Temp. disabled '-DOGS_BUILD_UTILS=ON ' + '-DOGS_CONAN_BUILD=missing ' + - '-DCMAKE_OSX_DEPLOYMENT_TARGET="10.14" ' + '-DCMAKE_OSX_DEPLOYMENT_TARGET="10.14" ' + + '-DOGS_USE_NETCDF=ON ' } build { target="package" diff --git a/Tests/Data/FileConverter/slim_100897_198.nc b/Tests/Data/FileConverter/slim_100897_198.nc new file mode 100644 index 0000000000000000000000000000000000000000..0127d346117a115ba66580a707cfa8ca6e2a7cec --- /dev/null +++ b/Tests/Data/FileConverter/slim_100897_198.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb278e02f075784b28241b476a1745dcad93131d9751415afbf00b9657e01718 +size 5352788 diff --git a/Tests/Data/FileConverter/slim_100897_198.vtu b/Tests/Data/FileConverter/slim_100897_198.vtu new file mode 100644 index 0000000000000000000000000000000000000000..a81f426f1ecf049384fd858b906659e4f4ec6463 --- /dev/null +++ b/Tests/Data/FileConverter/slim_100897_198.vtu @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ceec1ea5ea0967e7111c945e5e41032a8351a60bddd5ecc8675ef40ab46432e +size 4462866 diff --git a/Tests/Data/FileConverter/sresa1b_ncar_ccsm3-example.nc b/Tests/Data/FileConverter/sresa1b_ncar_ccsm3-example.nc new file mode 100644 index 0000000000000000000000000000000000000000..4b6338fe11b086710877544dea3d3afe0f258853 --- /dev/null +++ b/Tests/Data/FileConverter/sresa1b_ncar_ccsm3-example.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c2047ee329654f3bebf0a4b0d99eada50f1183728a3c7c129b0bc50d404511b +size 2767916 diff --git a/Tests/Data/FileConverter/sresa1b_ncar_ccsm3-example.vtu b/Tests/Data/FileConverter/sresa1b_ncar_ccsm3-example.vtu new file mode 100644 index 0000000000000000000000000000000000000000..8be050ca1bb45388737a0bd6e7695862012c30e4 --- /dev/null +++ b/Tests/Data/FileConverter/sresa1b_ncar_ccsm3-example.vtu @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc1627a4ea12e0d223c59f7155ad22eda205fbd0a8f4745516d08cd9a80f63f +size 4644854 diff --git a/scripts/cmake/ConanSetup.cmake b/scripts/cmake/ConanSetup.cmake index 357811e777db642b32b0349872b7ca84cebe88d4..670a3094e4cc79140d500d9e94860fda7bc7b541 100644 --- a/scripts/cmake/ConanSetup.cmake +++ b/scripts/cmake/ConanSetup.cmake @@ -69,9 +69,9 @@ if(OGS_BUILD_GUI) set(CONAN_REQUIRES ${CONAN_REQUIRES} shapelib/1.3.0@bilke/stable libgeotiff/1.4.2@bilke/stable - qt/5.12.4@bincrafters/stable - # Overwrite VTK requirement to match Qt requirement - bzip2/1.0.8@conan/stable + qt/5.13.2@bincrafters/stable + # Fix for "Conflict in pcre/8.41@bincrafters/stable" + bzip2/1.0.8 ) set(CONAN_OPTIONS ${CONAN_OPTIONS} vtk:minimal=False @@ -88,13 +88,16 @@ if(OGS_BUILD_GUI) qt:with_sdl2=False qt:with_sqlite3=False ) + if(MSVC) + set(CONAN_OPTIONS ${CONAN_OPTIONS} qt:with_harfbuzz=False) + endif() endif() if(OGS_USE_NETCDF) set(CONAN_REQUIRES ${CONAN_REQUIRES} netcdf-cxx/4.3.1@bilke/testing) endif() -conan_check(VERSION 1.19.2) +conan_check(VERSION 1.20.0) message(STATUS "Third-party libraries:") foreach(LIB ${OGS_LIBS}) diff --git a/scripts/cmake/Find.cmake b/scripts/cmake/Find.cmake index 254955b29e1de49396a15a94976ca9faf7c572a8..2c7bfe9a1566ad1982ac237e698c955902f400cd 100644 --- a/scripts/cmake/Find.cmake +++ b/scripts/cmake/Find.cmake @@ -106,6 +106,7 @@ if(OGS_USE_NETCDF) set(NETCDF_ROOT ${CONAN_NETCDF-C_ROOT}) set(NETCDF_CXX_ROOT ${CONAN_NETCDF-CXX_ROOT}) find_package(NetCDF REQUIRED) + find_package(HDF5 REQUIRED COMPONENTS C HL) endif() # lapack diff --git a/scripts/docker/Dockerfile.clang.full b/scripts/docker/Dockerfile.clang.full index a73db1a224c728335b69d7d23df732757d50dcf9..e4fbb7acde8278c5be1bb0684c23a8f4604165e9 100644 --- a/scripts/docker/Dockerfile.clang.full +++ b/scripts/docker/Dockerfile.clang.full @@ -31,8 +31,10 @@ RUN apt-get update -y && \ wget && \ rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh && \ + mkdir -p /usr/local && \ /bin/sh /var/tmp/cmake-3.12.4-Linux-x86_64.sh --prefix=/usr/local --skip-license && \ rm -rf /var/tmp/cmake-3.12.4-Linux-x86_64.sh +ENV PATH=/usr/local/bin:$PATH # OGS base building block # Python @@ -84,12 +86,12 @@ RUN apt-get update -y && \ python3-setuptools \ python3-wheel && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install conan==1.19.2 +RUN pip3 install conan==1.20.5 RUN mkdir -p /opt/conan && \ chmod 777 /opt/conan ENV CONAN_USER_HOME=/opt/conan LABEL org.opengeosys.pm=conan \ - org.opengeosys.pm.conan.version=1.19.2 + org.opengeosys.pm.conan.version=1.20.5 LABEL org.opengeosys.pm.conan.user_home=/opt/conan # Include-what-you-use for clang version 8 @@ -100,11 +102,13 @@ RUN apt-get update -y && \ llvm-8-dev \ zlib1g-dev && \ rm -rf /var/lib/apt/lists/* +# https://github.com/include-what-you-use/include-what-you-use/archive/clang_8.0.tar.gz RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/include-what-you-use/include-what-you-use/archive/clang_8.0.tar.gz && \ mkdir -p /var/tmp && tar -x -f /var/tmp/clang_8.0.tar.gz -C /var/tmp -z && \ - mkdir -p /var/tmp/build && cd /var/tmp/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/iwyy -DIWYU_LLVM_ROOT_PATH=/usr/lib/llvm-8 /var/tmp/include-what-you-use-clang_8.0 && \ - cmake --build /var/tmp/build --target install -- -j$(nproc) && \ - rm -rf /var/tmp/clang_8.0.tar.gz /var/tmp/build /var/tmp/include-what-you-use-clang_8.0 + mkdir -p /var/tmp/include-what-you-use-clang_8.0/build && cd /var/tmp/include-what-you-use-clang_8.0/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/iwyy -D IWYU_LLVM_ROOT_PATH=/usr/lib/llvm-8 /var/tmp/include-what-you-use-clang_8.0 && \ + cmake --build /var/tmp/include-what-you-use-clang_8.0/build --target all -- -j$(nproc) && \ + cmake --build /var/tmp/include-what-you-use-clang_8.0/build --target install -- -j$(nproc) && \ + rm -rf /var/tmp/clang_8.0.tar.gz /var/tmp/include-what-you-use-clang_8.0 ENV PATH=/usr/local/iwyy/bin:$PATH # Package manager Conan building block diff --git a/scripts/docker/Dockerfile.gcc.full b/scripts/docker/Dockerfile.gcc.full index 85ee2907655f862e3ac13e4da89ffeb091970a7b..7f2a0b05c4b1620b310d36a9b3bde65295591efb 100644 --- a/scripts/docker/Dockerfile.gcc.full +++ b/scripts/docker/Dockerfile.gcc.full @@ -24,8 +24,10 @@ RUN apt-get update -y && \ wget && \ rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh && \ + mkdir -p /usr/local && \ /bin/sh /var/tmp/cmake-3.12.4-Linux-x86_64.sh --prefix=/usr/local --skip-license && \ rm -rf /var/tmp/cmake-3.12.4-Linux-x86_64.sh +ENV PATH=/usr/local/bin:$PATH # OGS base building block # Python @@ -77,20 +79,22 @@ RUN apt-get update -y && \ python3-setuptools \ python3-wheel && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install conan==1.19.2 +RUN pip3 install conan==1.20.5 RUN mkdir -p /opt/conan && \ chmod 777 /opt/conan ENV CONAN_USER_HOME=/opt/conan LABEL org.opengeosys.pm=conan \ - org.opengeosys.pm.conan.version=1.19.2 + org.opengeosys.pm.conan.version=1.20.5 LABEL org.opengeosys.pm.conan.user_home=/opt/conan # cppcheck version 1.87 +# https://github.com/danmar/cppcheck/archive/1.87.tar.gz RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/danmar/cppcheck/archive/1.87.tar.gz && \ mkdir -p /var/tmp && tar -x -f /var/tmp/1.87.tar.gz -C /var/tmp -z && \ - mkdir -p /var/tmp/build && cd /var/tmp/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local/cppcheck -DCMAKE_BUILD_TYPE=RELEASE /var/tmp/cppcheck-1.87 && \ - cmake --build /var/tmp/build --target install -- -j$(nproc) && \ - rm -rf /var/tmp/1.87.tar.gz /var/tmp/build /var/tmp/cppcheck-1.87 + mkdir -p /var/tmp/cppcheck-1.87/build && cd /var/tmp/cppcheck-1.87/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/cppcheck -D CMAKE_BUILD_TYPE=Release /var/tmp/cppcheck-1.87 && \ + cmake --build /var/tmp/cppcheck-1.87/build --target all -- -j$(nproc) && \ + cmake --build /var/tmp/cppcheck-1.87/build --target install -- -j$(nproc) && \ + rm -rf /var/tmp/1.87.tar.gz /var/tmp/cppcheck-1.87 ENV PATH=/usr/local/cppcheck/bin:$PATH RUN apt-get update -y && \ diff --git a/scripts/docker/Dockerfile.gcc.gui b/scripts/docker/Dockerfile.gcc.gui index b96f2497d188b121632606f84c3ece6df6ab77c3..62a986576f187be3e66194edb4ce9d937de6f656 100644 --- a/scripts/docker/Dockerfile.gcc.gui +++ b/scripts/docker/Dockerfile.gcc.gui @@ -24,8 +24,10 @@ RUN apt-get update -y && \ wget && \ rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh && \ + mkdir -p /usr/local && \ /bin/sh /var/tmp/cmake-3.12.4-Linux-x86_64.sh --prefix=/usr/local --skip-license && \ rm -rf /var/tmp/cmake-3.12.4-Linux-x86_64.sh +ENV PATH=/usr/local/bin:$PATH # OGS base building block # Python @@ -85,20 +87,22 @@ RUN apt-get update -y && \ python3-setuptools \ python3-wheel && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install conan==1.19.2 +RUN pip3 install conan==1.20.5 RUN mkdir -p /opt/conan && \ chmod 777 /opt/conan ENV CONAN_USER_HOME=/opt/conan LABEL org.opengeosys.pm=conan \ - org.opengeosys.pm.conan.version=1.19.2 + org.opengeosys.pm.conan.version=1.20.5 LABEL org.opengeosys.pm.conan.user_home=/opt/conan # cppcheck version 1.87 +# https://github.com/danmar/cppcheck/archive/1.87.tar.gz RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/danmar/cppcheck/archive/1.87.tar.gz && \ mkdir -p /var/tmp && tar -x -f /var/tmp/1.87.tar.gz -C /var/tmp -z && \ - mkdir -p /var/tmp/build && cd /var/tmp/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local/cppcheck -DCMAKE_BUILD_TYPE=RELEASE /var/tmp/cppcheck-1.87 && \ - cmake --build /var/tmp/build --target install -- -j$(nproc) && \ - rm -rf /var/tmp/1.87.tar.gz /var/tmp/build /var/tmp/cppcheck-1.87 + mkdir -p /var/tmp/cppcheck-1.87/build && cd /var/tmp/cppcheck-1.87/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/cppcheck -D CMAKE_BUILD_TYPE=Release /var/tmp/cppcheck-1.87 && \ + cmake --build /var/tmp/cppcheck-1.87/build --target all -- -j$(nproc) && \ + cmake --build /var/tmp/cppcheck-1.87/build --target install -- -j$(nproc) && \ + rm -rf /var/tmp/1.87.tar.gz /var/tmp/cppcheck-1.87 ENV PATH=/usr/local/cppcheck/bin:$PATH # pip