From 834f5b256bdd915e8c6ce59da3c244e02b2cc4ab Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Thu, 12 Sep 2024 12:53:36 +0700 Subject: [PATCH 01/12] added core package ext-lib --- admin.py | 83 + requirements.txt | 3 + src/cplus_plugin/__init__.py | 13 + .../api/scenario_task_api_client.py | 23 +- src/cplus_plugin/gui/qgis_cplus_main.py | 113 +- src/cplus_plugin/tasks.py | 2238 ----------------- 6 files changed, 199 insertions(+), 2274 deletions(-) create mode 100644 requirements.txt delete mode 100644 src/cplus_plugin/tasks.py diff --git a/admin.py b/admin.py index 17400b876..f3fbfde87 100644 --- a/admin.py +++ b/admin.py @@ -213,6 +213,7 @@ def build( if clean and output_directory.exists(): shutil.rmtree(str(output_directory), ignore_errors=True) output_directory.mkdir(parents=True, exist_ok=True) + plugin_setup(clean) copy_source_files(output_directory, tests=tests) icon_path = copy_icon(output_directory) if icon_path is None: @@ -552,5 +553,87 @@ def _get_latest_releases( return latest_stable, latest_experimental +############################################################################### +# Setup dependencies and install package +############################################################################### + + +def not_comments(lines, s, e): + """Return non comment line (does not start with #). + + :param lines: list of line + :type lines: list of str + + :param s: start index + :type s: int + + :param e: end index + :type e: int + + :return: list of line without commented lines + :rtype: list of str + """ + return [line for line in lines[s:e] if line[0] != "#"] + + +def read_requirements(): + """Return a list of runtime and list of test requirements.""" + with open("requirements.txt") as f: + lines = f.readlines() + lines = [line for line in [line.strip() for line in lines] if line] + + return not_comments(lines, 0, len(lines)), [] + + +def _safe_remove_folder(rootdir): + """ + Supports removing a folder that may have symlinks in it. + + Needed on windows to avoid removing the original files linked to within + each folder. + + :param rootdir: Root directory to clean. + :type rootdir: str + """ + rootdir = Path(rootdir) + if rootdir.is_symlink(): + rootdir.rmdir() + else: + folders = [path for path in Path(rootdir).iterdir() if path.is_dir()] + for folder in folders: + if folder.is_symlink(): + folder.rmdir() + else: + shutil.rmtree(folder) + files = [path for path in Path(rootdir).iterdir()] + for file in files: + file.unlink() + shutil.rmtree(rootdir) + + +@app.command() +def plugin_setup(clean=True, pip="pip"): + """install plugin dependencies. + + :param clean: Clean out dependencies first. + :type clean: bool + + :param pip: Path to pip, usually 'pip' or 'pip3'. + :type pip: str + """ + ext_libs = os.path.join(LOCAL_ROOT_DIR, "src", "cplus_plugin", "ext-libs") + + if clean and os.path.exists(ext_libs): + _safe_remove_folder(ext_libs) + + os.makedirs(ext_libs, exist_ok=True) + runtime, test = read_requirements() + + os.environ["PYTHONPATH"] = ext_libs + + for req in runtime + test: + subprocess.check_call([pip, "install", "--upgrade", "-t", ext_libs, req]) + + if __name__ == "__main__": app() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..684fb77c4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# cplus-core +# git+https://github.com/kartoza/cplus-core.git@v0.0.3 +git+https://github.com/kartoza/cplus-core.git@feat-refactor-conf \ No newline at end of file diff --git a/src/cplus_plugin/__init__.py b/src/cplus_plugin/__init__.py index 225cbc87a..beaf67aa4 100644 --- a/src/cplus_plugin/__init__.py +++ b/src/cplus_plugin/__init__.py @@ -30,6 +30,19 @@ sys.path.append(LIB_DIR) +def _add_at_front_of_path(d): + """add a folder at front of path""" + sys.path, remainder = sys.path[:1], sys.path[1:] + site.addsitedir(d) + sys.path.extend(remainder) + + +# init ext-libs directory +plugin_dir = os.path.dirname(os.path.realpath(__file__)) +# Put ext-libs folder near the front of the path (important on Linux) +_add_at_front_of_path(str(Path(plugin_dir) / "ext-libs")) + + # noinspection PyPep8Naming def classFactory(iface): # pylint: disable=invalid-name """Load QgisCplus class diff --git a/src/cplus_plugin/api/scenario_task_api_client.py b/src/cplus_plugin/api/scenario_task_api_client.py index 1d82f502c..9c0b16581 100644 --- a/src/cplus_plugin/api/scenario_task_api_client.py +++ b/src/cplus_plugin/api/scenario_task_api_client.py @@ -15,7 +15,7 @@ from ..conf import settings_manager, Settings from ..models.base import Activity, NcsPathway, Scenario from ..models.base import ScenarioResult -from ..tasks import ScenarioAnalysisTask +from cplus_core.analysis import ScenarioAnalysisTask, TaskConfig from ..utils import FileUtils, CustomJsonEncoder, todict @@ -60,23 +60,8 @@ class ScenarioAnalysisTaskApiClient(ScenarioAnalysisTask): :type scenario: Scenario """ - def __init__( - self, - analysis_scenario_name: str, - analysis_scenario_description: str, - analysis_activities: typing.List[Activity], - analysis_priority_layers_groups: typing.List[dict], - analysis_extent: typing.List[float], - scenario: Scenario, - ): - super().__init__( - analysis_scenario_name, - analysis_scenario_description, - analysis_activities, - analysis_priority_layers_groups, - analysis_extent, - scenario, - ) + def __init__(self, task_config: TaskConfig): + super().__init__(task_config) self.total_file_upload_size = 0 self.total_file_upload_chunks = 0 self.uploaded_chunks = 0 @@ -136,7 +121,7 @@ def run(self) -> bool: :rtype: bool """ self.request = CplusApiRequest() - self.scenario_directory = self.get_scenario_directory() + self.scenario_directory = self.task_config.base_dir FileUtils.create_new_dir(self.scenario_directory) try: diff --git a/src/cplus_plugin/gui/qgis_cplus_main.py b/src/cplus_plugin/gui/qgis_cplus_main.py index b4640cbff..984e19653 100644 --- a/src/cplus_plugin/gui/qgis_cplus_main.py +++ b/src/cplus_plugin/gui/qgis_cplus_main.py @@ -12,6 +12,7 @@ from dateutil import tz from functools import partial from pathlib import Path +import traceback from qgis.PyQt import ( QtCore, @@ -94,7 +95,7 @@ ) from ..lib.reports.manager import report_manager, ReportManager from ..models.base import Scenario, ScenarioResult, ScenarioState, SpatialExtent -from ..tasks import ScenarioAnalysisTask +from cplus_core.analysis import ScenarioAnalysisTask, TaskConfig from ..utils import open_documentation, tr, log, FileUtils, write_to_file @@ -1283,6 +1284,94 @@ def prepare_message_bar(self): ) self.dock_widget_contents.layout().insertLayout(0, self.grid_layout) + def get_scenario_directory(self) -> str: + """Generate scenario directory for current task. + + :return: Path to scenario directory + :rtype: str + """ + base_dir = settings_manager.get_value(Settings.BASE_DIR) + return os.path.join( + f"{base_dir}", + "scenario_" f'{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}', + ) + + def create_task_config(self, scenario: Scenario) -> TaskConfig: + """Create task config from scenario and settings_manager. + + :param scenario: Scenario object + :type scenario: Scenario + + :return: config for scenario analysis task + :rtype: TaskConfig + """ + task_config = TaskConfig( + scenario.name, + scenario.description, + scenario.extent.bbox, + scenario.activities, + settings_manager.get_priority_layers(), + scenario.priority_layer_groups, + str(scenario.uuid), + settings_manager.get_value( + Settings.SNAPPING_ENABLED, default=False, setting_type=bool + ), + settings_manager.get_value(Settings.RESAMPLING_METHOD, default=0), + settings_manager.get_value( + Settings.RESCALE_VALUES, default=False, setting_type=bool + ), + settings_manager.get_value(Settings.PATHWAY_SUITABILITY_INDEX, default=0), + settings_manager.get_value(Settings.CARBON_COEFFICIENT, default=0.0), + settings_manager.get_value( + Settings.SIEVE_ENABLED, default=False, setting_type=bool + ), + settings_manager.get_value(Settings.SIEVE_THRESHOLD, default=10.0), + settings_manager.get_value( + Settings.NCS_WITH_CARBON, default=True, setting_type=bool + ), + settings_manager.get_value( + Settings.LANDUSE_PROJECT, default=True, setting_type=bool + ), + settings_manager.get_value( + Settings.LANDUSE_NORMALIZED, default=True, setting_type=bool + ), + settings_manager.get_value( + Settings.LANDUSE_WEIGHTED, default=True, setting_type=bool + ), + settings_manager.get_value( + Settings.HIGHEST_POSITION, default=True, setting_type=bool + ), + self.get_scenario_directory(), + ) + task_config.scenario = scenario + return task_config + + def on_log_message( + self, + message: str, + name: str = "qgis_cplus", + info: bool = True, + notify: bool = True, + ): + """Logs the message into QGIS logs using qgis_cplus as the default + log instance. + If notify_user is True, user will be notified about the log. + + :param message: The log message + :type message: str + + :param name: Name of te log instance, qgis_cplus is the default + :type message: str + + :param info: Whether the message is about info or a + warning + :type info: bool + + :param notify: Whether to notify user about the log + :type notify: bool + """ + log(message, name=name, info=info, notify=notify) + def run_analysis(self): """Runs the plugin analysis Creates new QgsTask, progress dialog and report manager @@ -1417,6 +1506,7 @@ def run_analysis(self): weighted_activities=[], priority_layer_groups=self.analysis_priority_layers_groups, ) + task_config = self.create_task_config(scenario) self.processing_cancelled = False @@ -1491,23 +1581,9 @@ def run_analysis(self): transformed_extent.yMaximum(), ] if self.processing_type.isChecked(): - analysis_task = ScenarioAnalysisTaskApiClient( - self.analysis_scenario_name, - self.analysis_scenario_description, - self.analysis_activities, - self.analysis_priority_layers_groups, - self.analysis_extent, - scenario, - ) + analysis_task = ScenarioAnalysisTaskApiClient(task_config) else: - analysis_task = ScenarioAnalysisTask( - self.analysis_scenario_name, - self.analysis_scenario_description, - self.analysis_activities, - self.analysis_priority_layers_groups, - self.analysis_extent, - scenario, - ) + analysis_task = ScenarioAnalysisTask(task_config) progress_changed = partial(self.update_progress_bar, progress_dialog) analysis_task.custom_progress_changed.connect(progress_changed) @@ -1520,6 +1596,8 @@ def run_analysis(self): analysis_task.info_message_changed.connect(self.show_message) + analysis_task.log_received.connect(self.on_log_message) + self.current_analysis_task = analysis_task progress_dialog.analysis_task = analysis_task @@ -1561,6 +1639,7 @@ def run_analysis(self): ', error message "{}"'.format(err) ) ) + log(traceback.format_exc()) def task_terminated( self, task: typing.Union[ScenarioAnalysisTask, ScenarioAnalysisTaskApiClient] diff --git a/src/cplus_plugin/tasks.py b/src/cplus_plugin/tasks.py deleted file mode 100644 index 1fb02a550..000000000 --- a/src/cplus_plugin/tasks.py +++ /dev/null @@ -1,2238 +0,0 @@ -# coding=utf-8 -""" - Plugin tasks related to the scenario analysis - -""" -import datetime -import math -import os -import uuid -import typing -from pathlib import Path - -from qgis import processing -from qgis.PyQt import QtCore -from qgis.core import ( - Qgis, - QgsCoordinateReferenceSystem, - QgsProcessing, - QgsProcessingContext, - QgsProcessingFeedback, - QgsRasterLayer, - QgsRectangle, - QgsVectorLayer, - QgsWkbTypes, -) -from qgis.core import QgsTask - -from .conf import settings_manager, Settings -from .definitions.defaults import ( - SCENARIO_OUTPUT_FILE_NAME, -) -from .models.base import ScenarioResult, Activity -from .models.helpers import clone_activity -from .resources import * -from .utils import align_rasters, clean_filename, tr, log, FileUtils - - -class ScenarioAnalysisTask(QgsTask): - """Prepares and runs the scenario analysis""" - - status_message_changed = QtCore.pyqtSignal(str) - info_message_changed = QtCore.pyqtSignal(str, int) - - custom_progress_changed = QtCore.pyqtSignal(float) - - def __init__( - self, - analysis_scenario_name, - analysis_scenario_description, - analysis_activities, - analysis_priority_layers_groups, - analysis_extent, - scenario, - ): - super().__init__() - self.analysis_scenario_name = analysis_scenario_name - self.analysis_scenario_description = analysis_scenario_description - - self.analysis_activities = analysis_activities - self.analysis_priority_layers_groups = analysis_priority_layers_groups - self.analysis_extent = analysis_extent - self.analysis_extent_string = None - - self.analysis_weighted_activities = [] - self.scenario_result = None - self.scenario_directory = None - - self.success = True - self.output = None - self.error = None - self.status_message = None - - self.info_message = None - - self.processing_cancelled = False - self.feedback = QgsProcessingFeedback() - self.processing_context = QgsProcessingContext() - - self.scenario = scenario - - def get_settings_value(self, name: str, default=None, setting_type=None): - """Gets value of the setting with the passed name. - - :param name: Name of setting key - :type name: str - - :param default: Default value returned when the setting key does not exist - :type default: Any - - :param setting_type: Type of the store setting - :type setting_type: Any - - :returns: Value of the setting - :rtype: Any - """ - return settings_manager.get_value(name, default, setting_type) - - def get_scenario_directory(self) -> str: - """Generate scenario directory for current task. - - :return: Path to scenario directory - :rtype: str - """ - base_dir = self.get_settings_value(Settings.BASE_DIR) - return os.path.join( - f"{base_dir}", - "scenario_" f'{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}', - ) - - def get_priority_layer(self, identifier) -> typing.Dict: - """Retrieves the priority layer that matches the passed identifier. - - :param identifier: Priority layers identifier - :type identifier: uuid.UUID - - :returns: Priority layer dict - :rtype: dict - """ - return settings_manager.get_priority_layer(identifier) - - def get_activity(self, activity_uuid) -> typing.Union[Activity, None]: - """Gets an activity object matching the given unique - identifier. - - :param activity_uuid: Unique identifier of the - activity object. - :type activity_uuid: str - - :returns: Returns the activity object matching the given - identifier else None if not found. - :rtype: Activity - """ - return settings_manager.get_activity(activity_uuid) - - def get_priority_layers(self) -> typing.List: - """Gets all the available priority layers in the plugin. - - :returns: Priority layers list - :rtype: list - """ - return settings_manager.get_priority_layers() - - def get_masking_layers(self) -> typing.List: - """Gets all the masking layers. - - :return: List of masking layer paths - :rtype: list - """ - masking_layers_paths = self.get_settings_value( - Settings.MASK_LAYERS_PATHS, default=None - ) - masking_layers = masking_layers_paths.split(",") if masking_layers_paths else [] - masking_layers.remove("") if "" in masking_layers else None - return masking_layers - - def cancel_task(self, exception=None): - """Cancel current task. - - :param exception: Exception if stopped with error, defaults to None - :type exception: Any, optional - """ - self.error = exception - self.cancel() - - def log_message( - self, - message: str, - name: str = "qgis_cplus", - info: bool = True, - notify: bool = True, - ): - """Logs the message into QGIS logs using qgis_cplus as the default - log instance. - If notify_user is True, user will be notified about the log. - - :param message: The log message - :type message: str - - :param name: Name of te log instance, qgis_cplus is the default - :type message: str - - :param info: Whether the message is about info or a - warning - :type info: bool - - :param notify: Whether to notify user about the log - :type notify: bool - """ - log(message, name=name, info=info, notify=notify) - - def on_terminated(self): - """Called when the task is terminated.""" - message = "Processing has been cancelled by the user." - if self.error: - message = f"Problem in running scenario analysis: {self.error}" - self.set_status_message(tr(message)) - self.log_message(message) - - def run(self): - """Runs the main scenario analysis task operations""" - - self.scenario_directory = self.get_scenario_directory() - - FileUtils.create_new_dir(self.scenario_directory) - - selected_pathway = None - pathway_found = False - - for activity in self.analysis_activities: - if pathway_found: - break - for pathway in activity.pathways: - if pathway is not None: - pathway_found = True - selected_pathway = pathway - break - - target_layer = QgsRasterLayer(selected_pathway.path, selected_pathway.name) - - dest_crs = ( - target_layer.crs() - if selected_pathway and selected_pathway.path - else QgsCoordinateReferenceSystem("EPSG:4326") - ) - - processing_extent = QgsRectangle( - float(self.analysis_extent.bbox[0]), - float(self.analysis_extent.bbox[2]), - float(self.analysis_extent.bbox[1]), - float(self.analysis_extent.bbox[3]), - ) - - snapped_extent = self.align_extent(target_layer, processing_extent) - - extent_string = ( - f"{snapped_extent.xMinimum()},{snapped_extent.xMaximum()}," - f"{snapped_extent.yMinimum()},{snapped_extent.yMaximum()}" - f" [{dest_crs.authid()}]" - ) - - self.log_message( - "Original area of interest extent: " - f"{processing_extent.asWktPolygon()} \n" - ) - self.log_message( - "Snapped area of interest extent " f"{snapped_extent.asWktPolygon()} \n" - ) - # Run pathways layers snapping using a specified reference layer - - snapping_enabled = self.get_settings_value( - Settings.SNAPPING_ENABLED, default=False, setting_type=bool - ) - reference_layer = self.get_settings_value(Settings.SNAP_LAYER, default="") - reference_layer_path = Path(reference_layer) - if ( - snapping_enabled - and os.path.exists(reference_layer) - and reference_layer_path.is_file() - ): - self.snap_analysis_data( - self.analysis_activities, - extent_string, - ) - - # Preparing all the pathways by adding them together with - # their carbon layers before creating - # their respective activities. - - save_output = self.get_settings_value( - Settings.NCS_WITH_CARBON, default=True, setting_type=bool - ) - - self.run_pathways_analysis( - self.analysis_activities, - extent_string, - temporary_output=not save_output, - ) - - # Normalizing all the activities pathways using the carbon coefficient and - # the pathway suitability index - - self.run_pathways_normalization( - self.analysis_activities, - extent_string, - ) - - # Creating activities from the normalized pathways - - save_output = self.get_settings_value( - Settings.LANDUSE_PROJECT, default=True, setting_type=bool - ) - - self.run_activities_analysis( - self.analysis_activities, - extent_string, - temporary_output=not save_output, - ) - - # Run masking of the activities layers - masking_layers = self.get_masking_layers() - - if masking_layers: - self.run_activities_masking( - self.analysis_activities, - masking_layers, - extent_string, - ) - - # TODO enable the sieve functionality - sieve_enabled = self.get_settings_value( - Settings.SIEVE_ENABLED, default=False, setting_type=bool - ) - - if sieve_enabled: - self.run_activities_sieve( - self.analysis_activities, - ) - - # After creating activities, we normalize them using the same coefficients - # used in normalizing their respective pathways. - - save_output = self.get_settings_value( - Settings.LANDUSE_NORMALIZED, default=True, setting_type=bool - ) - - self.run_activities_normalization( - self.analysis_activities, - extent_string, - temporary_output=not save_output, - ) - - # Weighting the activities with their corresponding priority weighting layers - save_output = self.get_settings_value( - Settings.LANDUSE_WEIGHTED, default=True, setting_type=bool - ) - weighted_activities, result = self.run_activities_weighting( - self.analysis_activities, - self.analysis_priority_layers_groups, - extent_string, - temporary_output=not save_output, - ) - - self.analysis_weighted_activities = weighted_activities - self.scenario.weighted_activities = weighted_activities - - # Post weighting analysis - self.run_activities_cleaning( - weighted_activities, extent_string, temporary_output=not save_output - ) - - # The highest position tool analysis - save_output = self.get_settings_value( - Settings.HIGHEST_POSITION, default=True, setting_type=bool - ) - self.run_highest_position_analysis(temporary_output=not save_output) - - return True - - def finished(self, result: bool): - """Calls the handler responsible for doing post analysis workflow. - - :param result: Whether the run() operation finished successfully - :type result: bool - """ - if result: - self.log_message("Finished from the main task \n") - else: - self.log_message(f"Error from task scenario task {self.error}") - - def set_status_message(self, message: str): - """Set status message in progress dialog - - :param message: Message to be displayed - :type message: str - """ - self.status_message = message - self.status_message_changed.emit(self.status_message) - - def set_info_message(self, message: str, level=Qgis.Info): - """Set info message. - - :param message: Message - :type message: str - :param level: log level, defaults to Qgis.Info - :type level: int, optional - """ - self.info_message = message - self.info_message_changed.emit(self.info_message, level) - - def set_custom_progress(self, value: float): - """Set task progress value. - - :param value: Value to be set on the progress bar - :type value: float - """ - self.custom_progress = value - self.custom_progress_changed.emit(self.custom_progress) - - def update_progress(self, value): - """Sets the value of the task progress - - :param value: Value to be set on the progress bar - :type value: float - """ - if not self.processing_cancelled: - self.set_custom_progress(value) - else: - self.feedback = QgsProcessingFeedback() - self.processing_context = QgsProcessingContext() - - def align_extent(self, raster_layer, target_extent): - """Snaps the passed extent to the activities pathway layer pixel bounds - - :param raster_layer: The target layer that the passed extent will be - aligned with - :type raster_layer: QgsRasterLayer - - :param target_extent: Spatial extent that will be used a target extent when - doing alignment. - :type target_extent: QgsRectangle - """ - - try: - raster_extent = raster_layer.extent() - - x_res = raster_layer.rasterUnitsPerPixelX() - y_res = raster_layer.rasterUnitsPerPixelY() - - left = raster_extent.xMinimum() + x_res * math.floor( - (target_extent.xMinimum() - raster_extent.xMinimum()) / x_res - ) - right = raster_extent.xMinimum() + x_res * math.ceil( - (target_extent.xMaximum() - raster_extent.xMinimum()) / x_res - ) - bottom = raster_extent.yMinimum() + y_res * math.floor( - (target_extent.yMinimum() - raster_extent.yMinimum()) / y_res - ) - top = raster_extent.yMaximum() - y_res * math.floor( - (raster_extent.yMaximum() - target_extent.yMaximum()) / y_res - ) - - return QgsRectangle(left, bottom, right, top) - - except Exception as e: - self.log_message( - tr( - f"Problem snapping area of " - f"interest extent, using the original extent," - f"{str(e)}" - ) - ) - - return target_extent - - def replace_nodata(self, layer_path, output_path, nodata_value): - """Adds nodata value info into the layer available - in the passed layer_path and save the layer in the passed output_path - path. - - The addition will replace any current nodata value available in - the input layer. - - :param layer_path: Input layer path - :type layer_path: str - - :param output_path: Output layer path - :type output_path: str - - :param nodata_value: Nodata value to be used - :type output_path: int - - :returns: Whether the task operations was successful - :rtype: bool - - """ - self.feedback = QgsProcessingFeedback() - self.feedback.progressChanged.connect(self.update_progress) - - try: - alg_params = { - "COPY_SUBDATASETS": False, - "DATA_TYPE": 6, # Float32 - "EXTRA": "", - "INPUT": layer_path, - "NODATA": None, - "OPTIONS": "", - "TARGET_CRS": None, - "OUTPUT": QgsProcessing.TEMPORARY_OUTPUT, - } - translate_output = processing.run( - "gdal:translate", - alg_params, - context=self.processing_context, - feedback=self.feedback, - is_child_algorithm=True, - ) - - alg_params = { - "DATA_TYPE": 0, # Use Input Layer Data Type - "EXTRA": "", - "INPUT": translate_output["OUTPUT"], - "MULTITHREADING": False, - "NODATA": -9999, - "OPTIONS": "", - "RESAMPLING": 0, # Nearest Neighbour - "SOURCE_CRS": None, - "TARGET_CRS": None, - "TARGET_EXTENT": None, - "TARGET_EXTENT_CRS": None, - "TARGET_RESOLUTION": None, - "OUTPUT": output_path, - } - outputs = processing.run( - "gdal:warpreproject", - alg_params, - context=self.processing_context, - feedback=self.feedback, - is_child_algorithm=True, - ) - - return outputs is not None - except Exception as e: - log(f"Problem replacing no data value from a snapping output, {e}") - - return False - - def run_pathways_analysis(self, activities, extent, temporary_output=False): - """Runs the required activity pathways analysis on the passed - activities. The analysis involves adding the pathways - carbon layers into their respective pathway layers. - - If a pathway layer has more than one carbon layer, the resulting - weighted pathway will contain the sum of the pathway layer values - with the average of the pathway carbon layers values. - - :param activities: List of the selected activities - :type activities: typing.List[Activity] - - :param extent: The selected extent from user - :type extent: SpatialExtent - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - """ - if self.processing_cancelled: - return False - - self.set_status_message(tr("Adding activity pathways with carbon layers")) - - pathways = [] - activities_paths = [] - - try: - for activity in activities: - if not activity.pathways and ( - activity.path is None or activity.path == "" - ): - self.set_info_message( - tr( - f"No defined activity pathways or an" - f" activity layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"No defined activity pathways or a " - f"activity layer for the activity {activity.name}" - ) - return False - - for pathway in activity.pathways: - if not (pathway in pathways): - pathways.append(pathway) - - if activity.path is not None and activity.path != "": - activities_paths.append(activity.path) - - if not pathways and len(activities_paths) > 0: - self.run_pathways_normalization(activities, extent) - return - - suitability_index = float( - self.get_settings_value(Settings.PATHWAY_SUITABILITY_INDEX, default=0) - ) - - carbon_coefficient = float( - self.get_settings_value(Settings.CARBON_COEFFICIENT, default=0.0) - ) - - for pathway in pathways: - basenames = [] - layers = [] - path_basename = Path(pathway.path).stem - layers.append(pathway.path) - - file_name = clean_filename(pathway.name.replace(" ", "_")) - - if suitability_index > 0: - basenames.append(f'{suitability_index} * "{path_basename}@1"') - else: - basenames.append(f'"{path_basename}@1"') - - carbon_names = [] - - if len(pathway.carbon_paths) <= 0: - continue - - new_carbon_directory = os.path.join( - self.scenario_directory, "pathways_carbon_layers" - ) - - FileUtils.create_new_dir(new_carbon_directory) - - output_file = os.path.join( - new_carbon_directory, f"{file_name}_{str(uuid.uuid4())[:4]}.tif" - ) - - for carbon_path in pathway.carbon_paths: - carbon_full_path = Path(carbon_path) - if not carbon_full_path.exists(): - continue - layers.append(carbon_path) - carbon_names.append(f'"{carbon_full_path.stem}@1"') - - if len(carbon_names) == 1 and carbon_coefficient > 0: - basenames.append(f"{carbon_coefficient} * ({carbon_names[0]})") - - # Setting up calculation to use carbon layers average when - # a pathway has more than one carbon layer. - if len(carbon_names) > 1 and carbon_coefficient > 0: - basenames.append( - f"{carbon_coefficient} * (" - f'({" + ".join(carbon_names)}) / ' - f"{len(pathway.carbon_paths)})" - ) - expression = " + ".join(basenames) - - if carbon_coefficient <= 0 and suitability_index <= 0: - self.run_pathways_normalization(activities, extent) - return - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - # Actual processing calculation - alg_params = { - "CELLSIZE": 0, - "CRS": None, - "EXPRESSION": expression, - "EXTENT": extent, - "LAYERS": layers, - "OUTPUT": output, - } - - self.log_message( - f"Used parameters for combining pathways" - f" and carbon layers generation: {alg_params} \n" - ) - - self.feedback = QgsProcessingFeedback() - - self.feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - results = processing.run( - "qgis:rastercalculator", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - - pathway.path = results["OUTPUT"] - except Exception as e: - self.log_message(f"Problem running pathway analysis, {e}") - self.cancel_task(e) - - return True - - def snap_analysis_data(self, activities, extent): - """Snaps the passed activities pathways, carbon layers and priority layers - to align with the reference layer set on the settings - manager. - - :param activities: List of the selected activities - :type activities: typing.List[Activity] - - :param extent: The selected extent from user - :type extent: list - """ - if self.processing_cancelled: - # Will not proceed if processing has been cancelled by the user - return False - - self.set_status_message( - tr( - "Snapping the selected activity pathways, " - "carbon layers and priority layers" - ) - ) - - pathways = [] - - try: - for activity in activities: - if not activity.pathways and ( - activity.path is None or activity.path == "" - ): - self.set_info_message( - tr( - f"No defined activity pathways or a" - f" activity layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"No defined activity pathways or a " - f"activity layer for the activity {activity.name}" - ) - return False - - for pathway in activity.pathways: - if not (pathway in pathways): - pathways.append(pathway) - - reference_layer_path = self.get_settings_value(Settings.SNAP_LAYER) - rescale_values = self.get_settings_value( - Settings.RESCALE_VALUES, default=False, setting_type=bool - ) - - resampling_method = self.get_settings_value( - Settings.RESAMPLING_METHOD, default=0 - ) - - if pathways is not None and len(pathways) > 0: - snapped_pathways_directory = os.path.join( - self.scenario_directory, "pathways" - ) - - FileUtils.create_new_dir(snapped_pathways_directory) - - for pathway in pathways: - pathway_layer = QgsRasterLayer(pathway.path, pathway.name) - nodata_value = pathway_layer.dataProvider().sourceNoDataValue(1) - - if self.processing_cancelled: - return False - - # carbon layer snapping - - self.log_message( - f"Snapping carbon layers from {pathway.name} pathway" - ) - - if ( - pathway.carbon_paths is not None - and len(pathway.carbon_paths) > 0 - ): - snapped_carbon_directory = os.path.join( - self.scenario_directory, "carbon_layers" - ) - - FileUtils.create_new_dir(snapped_carbon_directory) - - snapped_carbon_paths = [] - - for carbon_path in pathway.carbon_paths: - carbon_layer = QgsRasterLayer( - carbon_path, f"{str(uuid.uuid4())[:4]}" - ) - nodata_value_carbon = ( - carbon_layer.dataProvider().sourceNoDataValue(1) - ) - - carbon_output_path = self.snap_layer( - carbon_path, - reference_layer_path, - extent, - snapped_carbon_directory, - rescale_values, - resampling_method, - nodata_value_carbon, - ) - - if carbon_output_path: - snapped_carbon_paths.append(carbon_output_path) - else: - snapped_carbon_paths.append(carbon_path) - - pathway.carbon_paths = snapped_carbon_paths - - self.log_message(f"Snapping {pathway.name} pathway layer \n") - - # Pathway snapping - - output_path = self.snap_layer( - pathway.path, - reference_layer_path, - extent, - snapped_pathways_directory, - rescale_values, - resampling_method, - nodata_value, - ) - if output_path: - pathway.path = output_path - - for activity in activities: - self.log_message( - f"Snapping {len(activity.priority_layers)} " - f"priority weighting layers from activity {activity.name} with layers\n" - ) - - if ( - activity.priority_layers is not None - and len(activity.priority_layers) > 0 - ): - snapped_priority_directory = os.path.join( - self.scenario_directory, "priority_layers" - ) - - FileUtils.create_new_dir(snapped_priority_directory) - - priority_layers = [] - for priority_layer in activity.priority_layers: - if priority_layer is None: - continue - - priority_layer_settings = self.get_priority_layer( - priority_layer.get("uuid") - ) - if priority_layer_settings is None: - continue - - priority_layer_path = priority_layer_settings.get("path") - - if not Path(priority_layer_path).exists(): - priority_layers.append(priority_layer) - continue - - layer = QgsRasterLayer( - priority_layer_path, f"{str(uuid.uuid4())[:4]}" - ) - nodata_value_priority = layer.dataProvider().sourceNoDataValue( - 1 - ) - - priority_output_path = self.snap_layer( - priority_layer_path, - reference_layer_path, - extent, - snapped_priority_directory, - rescale_values, - resampling_method, - nodata_value_priority, - ) - - if priority_output_path: - priority_layer["path"] = priority_output_path - - priority_layers.append(priority_layer) - - activity.priority_layers = priority_layers - - except Exception as e: - self.log_message(f"Problem snapping layers, {e} \n") - self.cancel_task(e) - return False - - return True - - def snap_layer( - self, - input_path, - reference_path, - extent, - directory, - rescale_values, - resampling_method, - nodata_value, - ): - """Snaps the passed input layer using the reference layer and updates - the snap output no data value to be the same as the original input layer - no data value. - - :param input_path: Input layer source - :type input_path: str - - :param reference_path: Reference layer source - :type reference_path: str - - :param extent: Clip extent - :type extent: list - - :param directory: Absolute path of the output directory for the snapped - layers - :type directory: str - - :param rescale_values: Whether to rescale pixel values - :type rescale_values: bool - - :param resample_method: Method to use when resampling - :type resample_method: QgsAlignRaster.ResampleAlg - - :param nodata_value: Original no data value of the input layer - :type nodata_value: float - - """ - - input_result_path, reference_result_path = align_rasters( - input_path, - reference_path, - extent, - directory, - rescale_values, - resampling_method, - ) - - if input_result_path is not None: - result_path = Path(input_result_path) - - directory = result_path.parent - name = result_path.stem - - output_path = os.path.join(directory, f"{name}_final.tif") - - self.replace_nodata(input_result_path, output_path, nodata_value) - - return output_path - - def run_pathways_normalization(self, activities, extent, temporary_output=False): - """Runs the normalization on the activities pathways layers, - adjusting band values measured on different scale, the resulting scale - is computed using the below formula - Normalized_Pathway = (Carbon coefficient + Suitability index) * ( - (activity layer value) - (activity band minimum value)) / - (activity band maximum value - activity band minimum value)) - - If the carbon coefficient and suitability index are both zero then - the computation won't take them into account in the normalization - calculation. - - :param activities: List of the analyzed activities - :type activities: typing.List[Activity] - - :param extent: selected extent from user - :type extent: str - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - """ - if self.processing_cancelled: - # Will not proceed if processing has been cancelled by the user - return False - - self.set_status_message(tr("Normalization of pathways")) - - pathways = [] - activities_paths = [] - - try: - for activity in activities: - if not activity.pathways and ( - activity.path is None or activity.path == "" - ): - self.set_info_message( - tr( - f"No defined activity pathways or an" - f" activity layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"No defined activity pathways or an " - f"activity layer for the activity {activity.name}" - ) - - return False - - for pathway in activity.pathways: - if not (pathway in pathways): - pathways.append(pathway) - - if activity.path is not None and activity.path != "": - activities_paths.append(activity.path) - - if not pathways and len(activities_paths) > 0: - self.run_activities_analysis(activities, extent) - - return - - carbon_coefficient = float( - self.get_settings_value(Settings.CARBON_COEFFICIENT, default=0.0) - ) - - suitability_index = float( - self.get_settings_value(Settings.PATHWAY_SUITABILITY_INDEX, default=0) - ) - - normalization_index = carbon_coefficient + suitability_index - - for pathway in pathways: - layers = [] - normalized_pathways_directory = os.path.join( - self.scenario_directory, "normalized_pathways" - ) - FileUtils.create_new_dir(normalized_pathways_directory) - file_name = clean_filename(pathway.name.replace(" ", "_")) - - output_file = os.path.join( - normalized_pathways_directory, - f"{file_name}_{str(uuid.uuid4())[:4]}.tif", - ) - - pathway_layer = QgsRasterLayer(pathway.path, pathway.name) - provider = pathway_layer.dataProvider() - band_statistics = provider.bandStatistics(1) - - min_value = band_statistics.minimumValue - max_value = band_statistics.maximumValue - - layer_name = Path(pathway.path).stem - - layers.append(pathway.path) - - self.log_message( - f"Found minimum {min_value} and " - f"maximum {max_value} for pathway " - f" \n" - ) - - if max_value < min_value: - raise Exception( - tr( - f"Pathway contains " - f"invalid minimum and maxmum band values" - ) - ) - - if normalization_index > 0: - expression = ( - f" {normalization_index} * " - f'("{layer_name}@1" - {min_value}) /' - f" ({max_value} - {min_value})" - ) - else: - expression = ( - f'("{layer_name}@1" - {min_value}) /' - f" ({max_value} - {min_value})" - ) - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - # Actual processing calculation - alg_params = { - "CELLSIZE": 0, - "CRS": None, - "EXPRESSION": expression, - "EXTENT": extent, - "LAYERS": layers, - "OUTPUT": output, - } - - self.log_message( - f"Used parameters for normalization of the pathways: {alg_params} \n" - ) - - self.feedback = QgsProcessingFeedback() - - self.feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - results = processing.run( - "qgis:rastercalculator", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - - # self.replace_nodata(results["OUTPUT"], output_file, -9999) - - pathway.path = results["OUTPUT"] - - except Exception as e: - self.log_message(f"Problem normalizing pathways layers, {e} \n") - self.cancel_task(e) - return False - - return True - - def run_activities_analysis(self, activities, extent, temporary_output=False): - """Runs the required activity analysis on the passed - activities pathways. The analysis is responsible for creating activities - layers from their respective pathways layers. - - :param activities: List of the selected activities - :type activities: typing.List[Activity] - - :param extent: selected extent from user - :type extent: SpatialExtent - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - """ - if self.processing_cancelled: - # Will not proceed if processing has been cancelled by the user - return False - - self.set_status_message(tr("Creating activity layers from pathways")) - - try: - for activity in activities: - activities_directory = os.path.join( - self.scenario_directory, "activities" - ) - FileUtils.create_new_dir(activities_directory) - file_name = clean_filename(activity.name.replace(" ", "_")) - - layers = [] - if not activity.pathways and ( - activity.path is None or activity.path == "" - ): - self.set_info_message( - tr( - f"No defined activity pathways or a" - f" activity layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"No defined activity pathways or an " - f"activity layer for the activity {activity.name}" - ) - - return False - - output_file = os.path.join( - activities_directory, f"{file_name}_{str(uuid.uuid4())[:4]}.tif" - ) - - # Due to the activities base class - # activity only one of the following blocks will be executed, - # the activity either contain a path or - # pathways - - if activity.path is not None and activity.path != "": - layers = [activity.path] - - for pathway in activity.pathways: - layers.append(pathway.path) - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - # Actual processing calculation - - alg_params = { - "IGNORE_NODATA": True, - "INPUT": layers, - "EXTENT": extent, - "OUTPUT_NODATA_VALUE": -9999, - "REFERENCE_LAYER": layers[0] if len(layers) > 0 else None, - "STATISTIC": 0, # Sum - "OUTPUT": output, - } - - self.log_message( - f"Used parameters for " f"activities generation: {alg_params} \n" - ) - - feedback = QgsProcessingFeedback() - - feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - results = processing.run( - "native:cellstatistics", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - activity.path = results["OUTPUT"] - - except Exception as e: - self.log_message(f"Problem creating activity layers, {e}") - self.cancel_task(e) - return False - - return True - - def run_activities_masking( - self, activities, masking_layers, extent, temporary_output=False - ): - """Applies the mask layers into the passed activities - - :param activities: List of the selected activities - :type activities: typing.List[Activity] - - :param masking_layers: Paths to the mask layers to be used - :type masking_layers: dict - - :param extent: selected extent from user - :type extent: str - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - """ - if self.processing_cancelled: - # Will not proceed if processing has been cancelled by the user - return False - - self.set_status_message(tr("Masking activities using the saved masked layers")) - - try: - if len(masking_layers) < 1: - return False - if len(masking_layers) > 1: - initial_mask_layer = self.merge_vector_layers(masking_layers) - else: - mask_layer_path = masking_layers[0] - initial_mask_layer = QgsVectorLayer(mask_layer_path, "mask", "ogr") - - if not initial_mask_layer.isValid(): - self.log_message( - f"Skipping activities masking " - f"using layer {mask_layer_path}, not a valid layer." - ) - return False - - if Qgis.versionInt() < 33000: - layer_check = initial_mask_layer.geometryType() == QgsWkbTypes.Polygon - else: - layer_check = ( - initial_mask_layer.geometryType() == Qgis.GeometryType.Polygon - ) - - if not layer_check: - self.log_message( - f"Skipping activities masking " - f"using layer {mask_layer_path}, not a polygon layer." - ) - return False - - extent_layer = self.layer_extent(extent) - mask_layer = self.mask_layer_difference(initial_mask_layer, extent_layer) - - if isinstance(mask_layer, str): - mask_layer = QgsVectorLayer(mask_layer, "ogr") - - if not mask_layer.isValid(): - self.log_message( - f"Skipping activities masking " - f"the created difference mask layer {mask_layer.source()}," - f" not a valid layer." - ) - return False - - for activity in activities: - if activity.path is None or activity.path == "": - if not self.processing_cancelled: - self.set_info_message( - tr( - f"Problem when masking activities, " - f"there is no map layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"Problem when masking activities, " - f"there is no map layer for the activity {activity.name}" - ) - else: - # If the user cancelled the processing - self.set_info_message( - tr(f"Processing has been cancelled by the user."), - level=Qgis.Critical, - ) - self.log_message(f"Processing has been cancelled by the user.") - - return False - - masked_activities_directory = os.path.join( - self.scenario_directory, "masked_activities" - ) - FileUtils.create_new_dir(masked_activities_directory) - file_name = clean_filename(activity.name.replace(" ", "_")) - - output_file = os.path.join( - masked_activities_directory, - f"{file_name}_{str(uuid.uuid4())[:4]}.tif", - ) - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - activity_layer = QgsRasterLayer(activity.path, "activity_layer") - - # Actual processing calculation - alg_params = { - "INPUT": activity.path, - "MASK": mask_layer, - "SOURCE_CRS": activity_layer.crs(), - "DESTINATION_CRS": activity_layer.crs(), - "TARGET_EXTENT": extent, - "OUTPUT": output, - "NO_DATA": -9999, - } - - self.log_message( - f"Used parameters for masking the activities: {alg_params} \n" - ) - - feedback = QgsProcessingFeedback() - - feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - results = processing.run( - "gdal:cliprasterbymasklayer", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - activity.path = results["OUTPUT"] - - except Exception as e: - self.log_message(f"Problem masking activities layers, {e} \n") - self.cancel_task(e) - return False - - return True - - def merge_vector_layers(self, layers): - """Merges the passed vector layers into a single layer - - :param layers: List of the vector layers paths - :type layers: typing.List[str] - - :return: Merged vector layer - :rtype: QgsMapLayer - """ - - input_map_layers = [] - - for layer_path in layers: - layer = QgsVectorLayer(layer_path, "mask", "ogr") - if layer.isValid(): - input_map_layers.append(layer) - else: - self.log_message( - f"Skipping invalid mask layer {layer_path} from masking." - ) - if len(input_map_layers) == 0: - return None - if len(input_map_layers) == 1: - return input_map_layers[0].source() - - self.set_status_message(tr("Merging mask layers")) - - # Actual processing calculation - alg_params = { - "LAYERS": input_map_layers, - "CRS": None, - "OUTPUT": QgsProcessing.TEMPORARY_OUTPUT, - } - - self.log_message(f"Used parameters for merging mask layers: {alg_params} \n") - - results = processing.run( - "native:mergevectorlayers", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - - return results["OUTPUT"] - - def layer_extent(self, extent): - """Creates a new vector layer contains has a - feature with geometry matching an extent parameter. - - :param extent: Extent parameter - :type extent: str - - :returns: Vector layer - :rtype: QgsVectorLayer - """ - - alg_params = { - "INPUT": extent, - "CRS": None, - "OUTPUT": QgsProcessing.TEMPORARY_OUTPUT, - } - - results = processing.run( - "native:extenttolayer", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - - return results["OUTPUT"] - - def mask_layer_difference(self, input_layer, overlay_layer): - """Creates a new vector layer that contains - difference of features between the two passed layers. - - :param input_layer: Input layer - :type input_layer: QgsVectorLayer - - :param overlay_layer: Target overlay layer - :type overlay_layer: QgsVectorLayer - - :returns: Vector layer - :rtype: QgsVectorLayer - """ - - alg_params = { - "INPUT": input_layer, - "OVERLAY": overlay_layer, - "OVERLAY_FIELDS_PREFIX": "", - "GRID_SIZE": None, - "OUTPUT": QgsProcessing.TEMPORARY_OUTPUT, - } - - results = processing.run( - "native:symmetricaldifference", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - - return results["OUTPUT"] - - def run_activities_sieve(self, models, temporary_output=False): - """Runs the sieve functionality analysis on the passed models layers, - removing the models layer polygons that are smaller than the provided - threshold size (in pixels) and replaces them with the pixel value of - the largest neighbour polygon. - - :param models: List of the analyzed activities - :type models: typing.List[ImplementationModel] - - :param extent: Selected area of interest extent - :type extent: str - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - """ - if self.processing_cancelled: - # Will not proceed if processing has been cancelled by the user - return False - - self.set_status_message(tr("Applying sieve function to the activities")) - - try: - for model in models: - if model.path is None or model.path == "": - if not self.processing_cancelled: - self.set_info_message( - tr( - f"Problem when running sieve function on models, " - f"there is no map layer for the model {model.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"Problem when running sieve function on models, " - f"there is no map layer for the model {model.name}" - ) - else: - # If the user cancelled the processing - self.set_info_message( - tr(f"Processing has been cancelled by the user."), - level=Qgis.Critical, - ) - self.log_message(f"Processing has been cancelled by the user.") - - return False - - sieved_ims_directory = os.path.join( - self.scenario_directory, "sieved_ims" - ) - FileUtils.create_new_dir(sieved_ims_directory) - file_name = clean_filename(model.name.replace(" ", "_")) - - output_file = os.path.join( - sieved_ims_directory, f"{file_name}_{str(uuid.uuid4())[:4]}.tif" - ) - - threshold_value = float( - self.get_settings_value(Settings.SIEVE_THRESHOLD, default=10.0) - ) - - mask_layer = self.get_settings_value( - Settings.SIEVE_MASK_PATH, default="" - ) - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - # Actual processing calculation - alg_params = { - "INPUT": model.path, - "THRESHOLD": threshold_value, - "MASK_LAYER": mask_layer, - "OUTPUT": output, - } - - self.log_message(f"Used parameters for sieving: {alg_params} \n") - - input_name = os.path.splitext(os.path.basename(model.path))[0] - - # Step 1: Create a binary mask from the original raster - binary_mask = processing.run( - "qgis:rastercalculator", - { - "CELLSIZE": 0, - "LAYERS": [model.path], - "CRS": None, - "EXPRESSION": f"{input_name}@1 > 0", - "OUTPUT": "TEMPORARY_OUTPUT", - }, - )["OUTPUT"] - - # feedback.pushInfo(f"binary mask {binary_mask}") - - # binary_mask_layer = QgsRasterLayer(binary_mask, 'binary') - - # QgsProject.instance().addMapLayer(binary_mask_layer) - - # Step 2: Run sieve analysis from on the binary mask - sieved_mask = processing.run( - "gdal:sieve", - { - "INPUT": binary_mask, - "THRESHOLD": threshold_value, - "EIGHT_CONNECTEDNESS": True, - "NO_MASK": True, - "MASK_LAYER": None, - "OUTPUT": "TEMPORARY_OUTPUT", - }, - context=self.processing_context, - feedback=self.feedback, - )["OUTPUT"] - - # feedback.pushInfo(f"sieved mask {sieved_mask}") - - # sieved_mask_layer = QgsRasterLayer(sieved_mask, 'sieved_mask') - - # QgsProject.instance().addMapLayer(sieved_mask_layer) - - expr = f"({os.path.splitext(os.path.basename(sieved_mask))[0]}@1 > 0) * {os.path.splitext(os.path.basename(sieved_mask))[0]}@1" - # feedback.pushInfo(f"used expression {expr}") - - # Step 3: Remove and convert any no data value to 0 - sieved_mask_clean = processing.run( - "qgis:rastercalculator", - { - "CELLSIZE": 0, - "LAYERS": [sieved_mask], - "CRS": None, - "EXPRESSION": expr, - "OUTPUT": "TEMPORARY_OUTPUT", - }, - context=self.processing_context, - feedback=self.feedback, - )["OUTPUT"] - - # feedback.pushInfo(f"sieved mask clean {sieved_mask_clean}") - - # sieved_mask_clean_layer = QgsRasterLayer(sieved_mask_clean, 'sieved_mask_clean') - - # QgsProject.instance().addMapLayer(sieved_mask_clean_layer) - - expr_2 = f"{input_name}@1 * {os.path.splitext(os.path.basename(sieved_mask_clean))[0]}@1" - - # feedback.pushInfo(f"Used expression 2 {expr_2}") - - # Step 4: Join the sieved mask with the original input layer to filter out the small areas - sieve_output = processing.run( - "qgis:rastercalculator", - { - "CELLSIZE": 0, - "LAYERS": [model.path, sieved_mask_clean], - "CRS": None, - "EXPRESSION": expr_2, - "OUTPUT": "TEMPORARY_OUTPUT", - }, - context=self.processing_context, - feedback=self.feedback, - )["OUTPUT"] - - # feedback.pushInfo(f"sieved output joined {sieve_output}") - - # sieve_output_layer = QgsRasterLayer(sieve_output, 'sieve_output') - - # QgsProject.instance().addMapLayer(sieve_output_layer) - - # expr_3 = f'if ( {os.path.splitext(os.path.basename(sieve_output))[0]}@1 <= 0, -9999, {os.path.splitext(os.path.basename(sieve_output))[0]}@1 )' - - # feedback.pushInfo(f"used expression 3 {expr_3}") - - # Step 5. Replace all 0 with -9999 using if ("combined@1" <= 0, -9999, "combined@1") - sieve_output_updated = processing.run( - "gdal:rastercalculator", - { - "INPUT_A": f"{sieve_output}", - "BAND_A": 1, - "FORMULA": "9999*(A<=0)*(-1)+A*(A>0)", - "NO_DATA": None, - "EXTENT_OPT": 0, - "PROJWIN": None, - "RTYPE": 5, - "OPTIONS": "", - "EXTRA": "", - "OUTPUT": "TEMPORARY_OUTPUT", - }, - context=self.processing_context, - feedback=self.feedback, - )["OUTPUT"] - - # feedback.pushInfo(f"sieved output updated {sieve_output_updated}") - - # sieve_output_updated_layer = QgsRasterLayer(sieve_output_updated, 'sieve_output_updated') - - # QgsProject.instance().addMapLayer(sieve_output_updated_layer) - - # Step 6. Run sum statistics with ignore no data values set to false and no data value of -9999 - results = processing.run( - "native:cellstatistics", - { - "INPUT": [sieve_output_updated], - "STATISTIC": 0, - "IGNORE_NODATA": False, - "REFERENCE_LAYER": sieve_output_updated, - "OUTPUT_NODATA_VALUE": -9999, - "OUTPUT": output, - }, - context=self.processing_context, - feedback=self.feedback, - ) - - # self.log_message( - # f"Used parameters for running sieve function to the models: {alg_params} \n" - # ) - - feedback = QgsProcessingFeedback() - - feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - model.path = results["OUTPUT"] - - except Exception as e: - self.log_message(f"Problem running sieve function on models layers, {e} \n") - self.cancel_task(e) - return False - - return True - - def run_activities_normalization(self, activities, extent, temporary_output=False): - """Runs the normalization analysis on the activities' layers, - adjusting band values measured on different scale, the resulting scale - is computed using the below formula - Normalized_activity = (Carbon coefficient + Suitability index) * ( - (Activity layer value) - (Activity band minimum value)) / - (Activity band maximum value - Activity band minimum value)) - - If the carbon coefficient and suitability index are both zero then - the computation won't take them into account in the normalization - calculation. - - :param activities: List of the analyzed activities - :type activities: typing.List[Activity] - - :param extent: Selected area of interest extent - :type extent: str - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - """ - if self.processing_cancelled: - # Will not proceed if processing has been cancelled by the user - return False - - self.set_status_message(tr("Normalization of the activities")) - - try: - for activity in activities: - if activity.path is None or activity.path == "": - if not self.processing_cancelled: - self.set_info_message( - tr( - f"Problem when running activities normalization, " - f"there is no map layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"Problem when running activities normalization, " - f"there is no map layer for the activity {activity.name}" - ) - else: - # If the user cancelled the processing - self.set_info_message( - tr(f"Processing has been cancelled by the user."), - level=Qgis.Critical, - ) - self.log_message(f"Processing has been cancelled by the user.") - - return False - - layers = [] - normalized_activities_directory = os.path.join( - self.scenario_directory, "normalized_activities" - ) - FileUtils.create_new_dir(normalized_activities_directory) - file_name = clean_filename(activity.name.replace(" ", "_")) - - output_file = os.path.join( - normalized_activities_directory, - f"{file_name}_{str(uuid.uuid4())[:4]}.tif", - ) - - activity_layer = QgsRasterLayer(activity.path, activity.name) - provider = activity_layer.dataProvider() - band_statistics = provider.bandStatistics(1) - - min_value = band_statistics.minimumValue - max_value = band_statistics.maximumValue - - self.log_message( - f"Found minimum {min_value} and " - f"maximum {max_value} for activity {activity.name} \n" - ) - - layer_name = Path(activity.path).stem - - layers.append(activity.path) - - carbon_coefficient = float( - self.get_settings_value(Settings.CARBON_COEFFICIENT, default=0.0) - ) - - suitability_index = float( - self.get_settings_value( - Settings.PATHWAY_SUITABILITY_INDEX, default=0 - ) - ) - - normalization_index = carbon_coefficient + suitability_index - - if normalization_index > 0: - expression = ( - f" {normalization_index} * " - f'("{layer_name}@1" - {min_value}) /' - f" ({max_value} - {min_value})" - ) - - else: - expression = ( - f'("{layer_name}@1" - {min_value}) /' - f" ({max_value} - {min_value})" - ) - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - # Actual processing calculation - alg_params = { - "CELLSIZE": 0, - "CRS": None, - "EXPRESSION": expression, - "EXTENT": extent, - "LAYERS": layers, - "OUTPUT": output, - } - - self.log_message( - f"Used parameters for normalization of the activities: {alg_params} \n" - ) - - feedback = QgsProcessingFeedback() - - feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - results = processing.run( - "qgis:rastercalculator", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - activity.path = results["OUTPUT"] - - except Exception as e: - self.log_message(f"Problem normalizing activity layers, {e} \n") - self.cancel_task(e) - return False - - return True - - def run_activities_weighting( - self, activities, priority_layers_groups, extent, temporary_output=False - ): - """Runs weighting analysis on the passed activities using - the corresponding activities weight layers. - - :param activities: List of the selected activities - :type activities: typing.List[Activity] - - :param priority_layers_groups: Used priority layers groups and their values - :type priority_layers_groups: dict - - :param extent: selected extent from user - :type extent: str - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: A tuple with the weighted activities outputs and - a value of whether the task operations was successful - :rtype: typing.Tuple[typing.List, bool] - """ - - if self.processing_cancelled: - return [], False - - self.set_status_message(tr(f"Weighting activities")) - - weighted_activities = [] - - try: - for original_activity in activities: - activity = clone_activity(original_activity) - - if activity.path is None or activity.path == "": - self.set_info_message( - tr( - f"Problem when running activities weighting, " - f"there is no map layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"Problem when running activities weighting, " - f"there is no map layer for the activity {activity.name}" - ) - - return [], False - - basenames = [] - layers = [] - - layers.append(activity.path) - basenames.append(f'"{Path(activity.path).stem}@1"') - - if not any(priority_layers_groups): - self.log_message( - f"There are no defined priority layers in groups," - f" skipping activities weighting step." - ) - self.run_activities_cleaning( - extent, temporary_output=temporary_output - ) - return - - if activity.priority_layers is None or activity.priority_layers is []: - self.log_message( - f"There are no associated " - f"priority weighting layers for activity {activity.name}" - ) - continue - - settings_activity = self.get_activity(str(activity.uuid)) - - for layer in settings_activity.priority_layers: - if layer is None: - continue - - settings_layer = self.get_priority_layer(layer.get("uuid")) - if settings_layer is None: - continue - - pwl = settings_layer.get("path") - - missing_pwl_message = ( - f"Path {pwl} for priority " - f"weighting layer {layer.get('name')} " - f"doesn't exist, skipping the layer " - f"from the activity {activity.name} weighting." - ) - if pwl is None: - self.log_message(missing_pwl_message) - continue - - pwl_path = Path(pwl) - - if not pwl_path.exists(): - self.log_message(missing_pwl_message) - continue - - path_basename = pwl_path.stem - - for priority_layer in self.get_priority_layers(): - if priority_layer.get("name") == layer.get("name"): - for group in priority_layer.get("groups", []): - value = group.get("value") - coefficient = float(value) - if coefficient > 0: - if pwl not in layers: - layers.append(pwl) - basenames.append( - f'({coefficient}*"{path_basename}@1")' - ) - - if basenames is []: - return [], True - - weighted_activities_directory = os.path.join( - self.scenario_directory, "weighted_activities" - ) - - FileUtils.create_new_dir(weighted_activities_directory) - - file_name = clean_filename(activity.name.replace(" ", "_")) - output_file = os.path.join( - weighted_activities_directory, - f"{file_name}_{str(uuid.uuid4())[:4]}.tif", - ) - expression = " + ".join(basenames) - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - # Actual processing calculation - alg_params = { - "CELLSIZE": 0, - "CRS": None, - "EXPRESSION": expression, - "EXTENT": extent, - "LAYERS": layers, - "OUTPUT": output, - } - - self.log_message( - f" Used parameters for calculating weighting activities {alg_params} \n" - ) - - feedback = QgsProcessingFeedback() - - feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return [], False - - results = processing.run( - "qgis:rastercalculator", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - activity.path = results["OUTPUT"] - - weighted_activities.append(activity) - - except Exception as e: - self.log_message(f"Problem weighting activities, {e}\n") - self.cancel_task(e) - return None, False - - return weighted_activities, True - - def run_activities_cleaning(self, activities, extent=None, temporary_output=False): - """Cleans the weighted activities replacing - zero values with no-data as they are not statistical meaningful for the - scenario analysis. - - :param extent: Selected extent from user - :type extent: str - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - """ - - if self.processing_cancelled: - return False - - self.set_status_message(tr("Updating weighted activity values")) - - try: - for activity in activities: - if activity.path is None or activity.path == "": - self.set_info_message( - tr( - f"Problem when running activity updates, " - f"there is no map layer for the activity {activity.name}" - ), - level=Qgis.Critical, - ) - self.log_message( - f"Problem when running activity updates, " - f"there is no map layer for the activity {activity.name}" - ) - - return False - - layers = [activity.path] - - file_name = clean_filename(activity.name.replace(" ", "_")) - - output_file = os.path.join( - self.scenario_directory, "weighted_activities" - ) - output_file = os.path.join( - output_file, f"{file_name}_{str(uuid.uuid4())[:4]}_cleaned.tif" - ) - - # Actual processing calculation - # The aim is to convert pixels values to no data, that is why we are - # using the sum operation with only one layer. - - output = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - alg_params = { - "IGNORE_NODATA": True, - "INPUT": layers, - "EXTENT": extent, - "OUTPUT_NODATA_VALUE": 0, - "REFERENCE_LAYER": layers[0] if len(layers) > 0 else None, - "STATISTIC": 0, # Sum - "OUTPUT": output, - } - - self.log_message( - f"Used parameters for " - f"updates on the weighted activities: {alg_params} \n" - ) - - feedback = QgsProcessingFeedback() - - feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - results = processing.run( - "native:cellstatistics", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - activity.path = results["OUTPUT"] - - except Exception as e: - self.log_message(f"Problem cleaning activities, {e}") - self.cancel_task(e) - return False - - return True - - def run_highest_position_analysis(self, temporary_output=False): - """Runs the highest position analysis which is last step - in scenario analysis. Uses the activities set by the current ongoing - analysis. - - :param temporary_output: Whether to save the processing outputs as temporary - files - :type temporary_output: bool - - :returns: Whether the task operations was successful - :rtype: bool - - """ - if self.processing_cancelled: - # Will not proceed if processing has been cancelled by the user - return False - - passed_extent_box = self.analysis_extent.bbox - passed_extent = QgsRectangle( - passed_extent_box[0], - passed_extent_box[2], - passed_extent_box[1], - passed_extent_box[3], - ) - - self.scenario_result = ScenarioResult( - scenario=self.scenario, scenario_directory=self.scenario_directory - ) - - try: - layers = {} - - self.set_status_message(tr("Calculating the highest position")) - - for activity in self.analysis_weighted_activities: - if activity.path is not None and activity.path != "": - raster_layer = QgsRasterLayer(activity.path, activity.name) - layers[activity.name] = ( - raster_layer if raster_layer is not None else None - ) - else: - for pathway in activity.pathways: - layers[activity.name] = QgsRasterLayer(pathway.path) - - source_crs = QgsCoordinateReferenceSystem("EPSG:4326") - dest_crs = list(layers.values())[0].crs() if len(layers) > 0 else source_crs - - extent_string = ( - f"{passed_extent.xMinimum()},{passed_extent.xMaximum()}," - f"{passed_extent.yMinimum()},{passed_extent.yMaximum()}" - f" [{dest_crs.authid()}]" - ) - - output_file = os.path.join( - self.scenario_directory, - f"{SCENARIO_OUTPUT_FILE_NAME}_{str(self.scenario.uuid)[:4]}.tif", - ) - - # Preparing the input rasters for the highest position - # analysis in a correct order - - activity_names = [ - activity.name for activity in self.analysis_weighted_activities - ] - all_activities = sorted( - self.analysis_weighted_activities, - key=lambda activity_instance: activity_instance.style_pixel_value, - ) - for index, activity in enumerate(all_activities): - activity.style_pixel_value = index + 1 - - all_activity_names = [activity.name for activity in all_activities] - sources = [] - - for activity_name in all_activity_names: - if activity_name in activity_names: - sources.append(layers[activity_name].source()) - - self.log_message( - f"Layers sources {[Path(source).stem for source in sources]}" - ) - - output_file = ( - QgsProcessing.TEMPORARY_OUTPUT if temporary_output else output_file - ) - - alg_params = { - "IGNORE_NODATA": True, - "INPUT_RASTERS": sources, - "EXTENT": extent_string, - "OUTPUT_NODATA_VALUE": -9999, - "REFERENCE_LAYER": list(layers.values())[0] - if len(layers) >= 1 - else None, - "OUTPUT": output_file, - } - - self.log_message( - f"Used parameters for highest position analysis {alg_params} \n" - ) - - self.feedback = QgsProcessingFeedback() - - self.feedback.progressChanged.connect(self.update_progress) - - if self.processing_cancelled: - return False - - self.output = processing.run( - "native:highestpositioninrasterstack", - alg_params, - context=self.processing_context, - feedback=self.feedback, - ) - - except Exception as err: - self.log_message( - tr( - "An error occurred when running task for " - 'scenario analysis, error message "{}"'.format(str(err)) - ) - ) - self.cancel_task(err) - return False - - return True From a9265af50b48abdbfd1d22d4124d4dfd84ebd4b3 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Thu, 12 Sep 2024 13:38:38 +0700 Subject: [PATCH 02/12] refactor base model import --- .gitignore | 2 +- .../api/scenario_task_api_client.py | 4 +- src/cplus_plugin/conf.py | 2 +- .../gui/activity_editor_dialog.py | 2 +- src/cplus_plugin/gui/activity_widget.py | 2 +- src/cplus_plugin/gui/component_item_model.py | 2 +- .../gui/financials/npv_manager_dialog.py | 2 +- .../gui/items_selection_dialog.py | 2 +- .../gui/map_repeat_item_widget.py | 2 +- .../gui/model_component_widget.py | 2 +- .../gui/ncs_pathway_editor_dialog.py | 2 +- src/cplus_plugin/gui/priority_group_dialog.py | 2 +- src/cplus_plugin/gui/priority_layer_dialog.py | 2 +- src/cplus_plugin/gui/qgis_cplus_main.py | 21 +- .../lib/reports/comparison_table.py | 2 +- src/cplus_plugin/lib/reports/generator.py | 2 +- src/cplus_plugin/lib/reports/layout_items.py | 2 +- src/cplus_plugin/lib/reports/manager.py | 2 +- src/cplus_plugin/lib/validation/manager.py | 2 +- src/cplus_plugin/lib/validation/validators.py | 2 +- src/cplus_plugin/models/base.py | 587 ------------------ src/cplus_plugin/models/financial.py | 2 +- src/cplus_plugin/models/helpers.py | 2 +- src/cplus_plugin/models/report.py | 2 +- src/cplus_plugin/models/validation.py | 2 +- test/model_data_for_testing.py | 2 +- test/test_data_model_helpers.py | 2 +- test/test_scenario_tasks.py | 2 +- 28 files changed, 38 insertions(+), 624 deletions(-) delete mode 100644 src/cplus_plugin/models/base.py diff --git a/.gitignore b/.gitignore index 337df5635..2c87750cc 100644 --- a/.gitignore +++ b/.gitignore @@ -133,6 +133,6 @@ dmypy.json build/ cplus_scenario_output.* [Skip output] -reports +# reports ext-libs/ \ No newline at end of file diff --git a/src/cplus_plugin/api/scenario_task_api_client.py b/src/cplus_plugin/api/scenario_task_api_client.py index 9c0b16581..aaf28d917 100644 --- a/src/cplus_plugin/api/scenario_task_api_client.py +++ b/src/cplus_plugin/api/scenario_task_api_client.py @@ -13,8 +13,8 @@ CHUNK_SIZE, ) from ..conf import settings_manager, Settings -from ..models.base import Activity, NcsPathway, Scenario -from ..models.base import ScenarioResult +from cplus_core.models.base import Activity, NcsPathway, Scenario +from cplus_core.models.base import ScenarioResult from cplus_core.analysis import ScenarioAnalysisTask, TaskConfig from ..utils import FileUtils, CustomJsonEncoder, todict diff --git a/src/cplus_plugin/conf.py b/src/cplus_plugin/conf.py index dafccb70d..e513a5106 100644 --- a/src/cplus_plugin/conf.py +++ b/src/cplus_plugin/conf.py @@ -28,7 +28,7 @@ UUID_ATTRIBUTE, ) from .definitions.defaults import PRIORITY_LAYERS -from .models.base import ( +from cplus_core.models.base import ( Activity, NcsPathway, PriorityLayerType, diff --git a/src/cplus_plugin/gui/activity_editor_dialog.py b/src/cplus_plugin/gui/activity_editor_dialog.py index 8f078b67f..556664024 100644 --- a/src/cplus_plugin/gui/activity_editor_dialog.py +++ b/src/cplus_plugin/gui/activity_editor_dialog.py @@ -29,7 +29,7 @@ ACTIVITY_SCENARIO_STYLE_ATTRIBUTE, ) from ..definitions.defaults import ICON_PATH, USER_DOCUMENTATION_SITE -from ..models.base import Activity +from cplus_core.models.base import Activity from ..utils import FileUtils, generate_random_color, open_documentation, tr WidgetUi, _ = loadUiType( diff --git a/src/cplus_plugin/gui/activity_widget.py b/src/cplus_plugin/gui/activity_widget.py index e3badb199..cef2938f6 100644 --- a/src/cplus_plugin/gui/activity_widget.py +++ b/src/cplus_plugin/gui/activity_widget.py @@ -19,7 +19,7 @@ ActivityComponentWidget, NcsComponentWidget, ) -from ..models.base import Activity, NcsPathway +from cplus_core.models.base import Activity, NcsPathway from ..utils import FileUtils diff --git a/src/cplus_plugin/gui/component_item_model.py b/src/cplus_plugin/gui/component_item_model.py index f1e139739..f424aa3ce 100644 --- a/src/cplus_plugin/gui/component_item_model.py +++ b/src/cplus_plugin/gui/component_item_model.py @@ -12,7 +12,7 @@ from qgis.PyQt import QtCore, QtGui -from ..models.base import ( +from cplus_core.models.base import ( BaseModelComponent, BaseModelComponentType, Activity, diff --git a/src/cplus_plugin/gui/financials/npv_manager_dialog.py b/src/cplus_plugin/gui/financials/npv_manager_dialog.py index 24f3a7b58..73b08296b 100644 --- a/src/cplus_plugin/gui/financials/npv_manager_dialog.py +++ b/src/cplus_plugin/gui/financials/npv_manager_dialog.py @@ -21,7 +21,7 @@ from ..component_item_model import ActivityItemModel from ...conf import settings_manager from ...definitions.defaults import ICON_PATH, USER_DOCUMENTATION_SITE -from ...models.base import Activity +from cplus_core.models.base import Activity from ...models.financial import ActivityNpv, ActivityNpvCollection, NpvParameters from .npv_financial_model import NpvFinancialModel from ...lib.financials import compute_discount_value diff --git a/src/cplus_plugin/gui/items_selection_dialog.py b/src/cplus_plugin/gui/items_selection_dialog.py index e0476a1fb..c8cadd2e4 100644 --- a/src/cplus_plugin/gui/items_selection_dialog.py +++ b/src/cplus_plugin/gui/items_selection_dialog.py @@ -11,7 +11,7 @@ from qgis.PyQt.uic import loadUiType -from ..models.base import Activity, PriorityLayer +from cplus_core.models.base import Activity, PriorityLayer from ..conf import settings_manager diff --git a/src/cplus_plugin/gui/map_repeat_item_widget.py b/src/cplus_plugin/gui/map_repeat_item_widget.py index 54aff0087..ad36f41b1 100644 --- a/src/cplus_plugin/gui/map_repeat_item_widget.py +++ b/src/cplus_plugin/gui/map_repeat_item_widget.py @@ -18,7 +18,7 @@ from ..conf import settings_manager from ..lib.reports.layout_items import CplusMapRepeatItem, CPLUS_MAP_REPEAT_ITEM_TYPE -from ..models.base import ModelComponentType +from cplus_core.models.base import ModelComponentType from ..utils import FileUtils, tr diff --git a/src/cplus_plugin/gui/model_component_widget.py b/src/cplus_plugin/gui/model_component_widget.py index e0439b02d..1a8e3d5a4 100644 --- a/src/cplus_plugin/gui/model_component_widget.py +++ b/src/cplus_plugin/gui/model_component_widget.py @@ -30,7 +30,7 @@ from .model_description_editor import ModelDescriptionEditorDialog from .ncs_pathway_editor_dialog import NcsPathwayEditorDialog from .pixel_value_editor_dialog import PixelValueEditorDialog -from ..models.base import Activity, NcsPathway +from cplus_core.models.base import Activity, NcsPathway from .validation.inspector_dialog import ValidationInspectorDialog from .validation.progress_dialog import ValidationProgressDialog from ..utils import FileUtils, log diff --git a/src/cplus_plugin/gui/ncs_pathway_editor_dialog.py b/src/cplus_plugin/gui/ncs_pathway_editor_dialog.py index 71e1b4673..731535429 100644 --- a/src/cplus_plugin/gui/ncs_pathway_editor_dialog.py +++ b/src/cplus_plugin/gui/ncs_pathway_editor_dialog.py @@ -16,7 +16,7 @@ from .carbon_item_model import CarbonLayerItem, CarbonLayerModel from ..conf import Settings, settings_manager from ..definitions.defaults import ICON_PATH, USER_DOCUMENTATION_SITE -from ..models.base import LayerType, NcsPathway +from cplus_core.models.base import LayerType, NcsPathway from ..utils import FileUtils, open_documentation, tr WidgetUi, _ = loadUiType( diff --git a/src/cplus_plugin/gui/priority_group_dialog.py b/src/cplus_plugin/gui/priority_group_dialog.py index 4d0b0cb0d..4bf1cea69 100644 --- a/src/cplus_plugin/gui/priority_group_dialog.py +++ b/src/cplus_plugin/gui/priority_group_dialog.py @@ -14,7 +14,7 @@ ) from qgis.PyQt.uic import loadUiType -from ..models.base import PriorityLayer +from cplus_core.models.base import PriorityLayer from ..conf import settings_manager from ..utils import FileUtils, open_documentation diff --git a/src/cplus_plugin/gui/priority_layer_dialog.py b/src/cplus_plugin/gui/priority_layer_dialog.py index f9f5734e2..f4e4c37bc 100644 --- a/src/cplus_plugin/gui/priority_layer_dialog.py +++ b/src/cplus_plugin/gui/priority_layer_dialog.py @@ -18,7 +18,7 @@ from ..conf import settings_manager, Settings from ..utils import FileUtils, open_documentation -from ..models.base import PriorityLayerType +from cplus_core.models.base import PriorityLayerType from ..definitions.defaults import ICON_PATH, PRIORITY_LAYERS, USER_DOCUMENTATION_SITE from ..definitions.constants import PRIORITY_LAYERS_SEGMENT, USER_DEFINED_ATTRIBUTE diff --git a/src/cplus_plugin/gui/qgis_cplus_main.py b/src/cplus_plugin/gui/qgis_cplus_main.py index 984e19653..89db504f4 100644 --- a/src/cplus_plugin/gui/qgis_cplus_main.py +++ b/src/cplus_plugin/gui/qgis_cplus_main.py @@ -67,7 +67,7 @@ from .scenario_dialog import ScenarioDialog -from ..models.base import ( +from cplus_core.models.base import ( PriorityLayerType, ) from ..models.financial import ActivityNpv @@ -94,7 +94,12 @@ USER_DOCUMENTATION_SITE, ) from ..lib.reports.manager import report_manager, ReportManager -from ..models.base import Scenario, ScenarioResult, ScenarioState, SpatialExtent +from cplus_core.models.base import ( + Scenario, + ScenarioResult, + ScenarioState, + SpatialExtent, +) from cplus_core.analysis import ScenarioAnalysisTask, TaskConfig from ..utils import open_documentation, tr, log, FileUtils, write_to_file @@ -1305,14 +1310,11 @@ def create_task_config(self, scenario: Scenario) -> TaskConfig: :return: config for scenario analysis task :rtype: TaskConfig """ - task_config = TaskConfig( - scenario.name, - scenario.description, - scenario.extent.bbox, - scenario.activities, + return TaskConfig( + scenario, settings_manager.get_priority_layers(), scenario.priority_layer_groups, - str(scenario.uuid), + settings_manager.get_all_activities(), settings_manager.get_value( Settings.SNAPPING_ENABLED, default=False, setting_type=bool ), @@ -1343,8 +1345,6 @@ def create_task_config(self, scenario: Scenario) -> TaskConfig: ), self.get_scenario_directory(), ) - task_config.scenario = scenario - return task_config def on_log_message( self, @@ -1580,6 +1580,7 @@ def run_analysis(self): transformed_extent.yMinimum(), transformed_extent.yMaximum(), ] + if self.processing_type.isChecked(): analysis_task = ScenarioAnalysisTaskApiClient(task_config) else: diff --git a/src/cplus_plugin/lib/reports/comparison_table.py b/src/cplus_plugin/lib/reports/comparison_table.py index 61c676587..e37d5897d 100644 --- a/src/cplus_plugin/lib/reports/comparison_table.py +++ b/src/cplus_plugin/lib/reports/comparison_table.py @@ -14,7 +14,7 @@ ) from qgis.PyQt import QtCore, QtGui -from ...models.base import ScenarioResult +from cplus_core.models.base import ScenarioResult from ...models.helpers import layer_from_scenario_result from ...models.report import ScenarioAreaInfo from ...utils import calculate_raster_value_area, log, tr diff --git a/src/cplus_plugin/lib/reports/generator.py b/src/cplus_plugin/lib/reports/generator.py index d3d2d6989..246e09f1c 100644 --- a/src/cplus_plugin/lib/reports/generator.py +++ b/src/cplus_plugin/lib/reports/generator.py @@ -59,7 +59,7 @@ PRIORITY_GROUP_WEIGHT_TABLE_ID, ) from .layout_items import BasicScenarioDetailsItem, CplusMapRepeatItem -from ...models.base import Activity, ScenarioResult +from cplus_core.models.base import Activity, ScenarioResult from ...models.helpers import extent_to_project_crs_extent from ...models.report import ( BaseReportContext, diff --git a/src/cplus_plugin/lib/reports/layout_items.py b/src/cplus_plugin/lib/reports/layout_items.py index 0845b1afe..7cb3c5c47 100644 --- a/src/cplus_plugin/lib/reports/layout_items.py +++ b/src/cplus_plugin/lib/reports/layout_items.py @@ -25,7 +25,7 @@ ) from qgis.PyQt import QtCore, QtGui -from ...models.base import ModelComponentType, ScenarioResult +from cplus_core.models.base import ModelComponentType, ScenarioResult from ...utils import FileUtils, get_report_font, log, tr diff --git a/src/cplus_plugin/lib/reports/manager.py b/src/cplus_plugin/lib/reports/manager.py index b94c4b3d9..37f962bb6 100644 --- a/src/cplus_plugin/lib/reports/manager.py +++ b/src/cplus_plugin/lib/reports/manager.py @@ -31,7 +31,7 @@ SCENARIO_ANALYSIS_TEMPLATE_NAME, SCENARIO_COMPARISON_TEMPLATE_NAME, ) -from ...models.base import Scenario, ScenarioResult +from cplus_core.models.base import Scenario, ScenarioResult from ...models.report import ( ReportContext, ReportResult, diff --git a/src/cplus_plugin/lib/validation/manager.py b/src/cplus_plugin/lib/validation/manager.py index edef41d69..6b1ed7023 100644 --- a/src/cplus_plugin/lib/validation/manager.py +++ b/src/cplus_plugin/lib/validation/manager.py @@ -12,7 +12,7 @@ from qgis.PyQt import QtCore, sip -from ...models.base import ModelComponentType, NcsPathway +from cplus_core.models.base import ModelComponentType, NcsPathway from ...models.helpers import clone_ncs_pathway from ...models.validation import SubmitResult, ValidationResult from .validators import NcsDataValidator diff --git a/src/cplus_plugin/lib/validation/validators.py b/src/cplus_plugin/lib/validation/validators.py index 352a50b5a..b0fceffe4 100644 --- a/src/cplus_plugin/lib/validation/validators.py +++ b/src/cplus_plugin/lib/validation/validators.py @@ -21,7 +21,7 @@ resolution_validation_config, ) from .feedback import ValidationFeedback -from ...models.base import LayerModelComponent, ModelComponentType, NcsPathway +from cplus_core.models.base import LayerModelComponent, ModelComponentType, NcsPathway from ...models.validation import ( RuleConfiguration, RuleInfo, diff --git a/src/cplus_plugin/models/base.py b/src/cplus_plugin/models/base.py deleted file mode 100644 index 2a1dfd6d2..000000000 --- a/src/cplus_plugin/models/base.py +++ /dev/null @@ -1,587 +0,0 @@ -# -*- coding: utf-8 -*- - -""" QGIS CPLUS plugin models. -""" - -import dataclasses -import datetime -from enum import Enum, IntEnum -import os.path -import typing -from uuid import UUID - -from qgis.core import ( - QgsColorBrewerColorRamp, - QgsColorRamp, - QgsCptCityColorRamp, - QgsFillSymbol, - QgsGradientColorRamp, - QgsLimitedRandomColorRamp, - QgsMapLayer, - QgsPresetSchemeColorRamp, - QgsRandomColorRamp, - QgsRasterLayer, - QgsVectorLayer, -) - -from ..definitions.constants import ( - COLOR_RAMP_PROPERTIES_ATTRIBUTE, - COLOR_RAMP_TYPE_ATTRIBUTE, - ACTIVITY_LAYER_STYLE_ATTRIBUTE, - ACTIVITY_SCENARIO_STYLE_ATTRIBUTE, -) - - -@dataclasses.dataclass -class SpatialExtent: - """Extent object that stores - the coordinates of the area of interest - """ - - bbox: typing.List[float] - - -class PRIORITY_GROUP(Enum): - """Represents priority groups types""" - - CARBON_IMPORTANCE = "Carbon importance" - BIODIVERSITY = "Biodiversity" - LIVELIHOOD = "Livelihood" - CLIMATE_RESILIENCE = "Climate Resilience" - ECOLOGICAL_INFRASTRUCTURE = "Ecological infrastructure" - POLICY = "Policy" - FINANCE_YEARS_EXPERIENCE = "Finance - Years Experience" - FINANCE_MARKET_TRENDS = "Finance - Market Trends" - FINANCE_NET_PRESENT_VALUE = "Finance - Net Present value" - FINANCE_CARBON = "Finance - Carbon" - - -@dataclasses.dataclass -class BaseModelComponent: - """Base class for common model item properties.""" - - uuid: UUID - name: str - description: str - - def __eq__(self, other: "BaseModelComponent") -> bool: - """Test equality of object with another BaseModelComponent - object using the attributes. - - :param other: BaseModelComponent object to compare with this object. - :type other: BaseModelComponent - - :returns: True if the all the attribute values match, else False. - :rtype: bool - """ - if self.uuid != other.uuid: - return False - - if self.name != other.name: - return False - - if self.description != other.description: - return False - - return True - - -BaseModelComponentType = typing.TypeVar( - "BaseModelComponentType", bound=BaseModelComponent -) - - -class LayerType(IntEnum): - """QGIS spatial layer type.""" - - RASTER = 0 - VECTOR = 1 - UNDEFINED = -1 - - -class ModelComponentType(Enum): - """Type of model component i.e. NCS pathway or - activity. - """ - - NCS_PATHWAY = "ncs_pathway" - ACTIVITY = "activity" - UNKNOWN = "unknown" - - @staticmethod - def from_string(str_enum: str) -> "ModelComponentType": - """Creates an enum from the corresponding string equivalent. - - :param str_enum: String representing the model component type. - :type str_enum: str - - :returns: Component type enum corresponding to the given - string else unknown if not found. - :rtype: ModelComponentType - """ - if str_enum.lower() == "ncs_pathway": - return ModelComponentType.NCS_PATHWAY - elif str_enum.lower() == "activity": - return ModelComponentType.ACTIVITY - - return ModelComponentType.UNKNOWN - - -@dataclasses.dataclass -class LayerModelComponent(BaseModelComponent): - """Base class for model components that support - a map layer. - """ - - path: str = "" - layer_type: LayerType = LayerType.UNDEFINED - user_defined: bool = False - - def __post_init__(self): - """Try to set the layer and layer type properties.""" - self.update_layer_type() - - def update_layer_type(self): - """Update the layer type if either the layer or - path properties have been set. - """ - layer = self.to_map_layer() - if layer is None: - return - - if not layer.isValid(): - return - - if isinstance(layer, QgsRasterLayer): - self.layer_type = LayerType.RASTER - - elif isinstance(layer, QgsVectorLayer): - self.layer_type = LayerType.VECTOR - - def to_map_layer(self) -> typing.Union[QgsMapLayer, None]: - """Constructs a map layer from the specified path. - - It will first check if the layer property has been set - else try to construct the layer from the path else return - None. - - :returns: Map layer corresponding to the set layer - property or specified path. - :rtype: QgsMapLayer - """ - if not os.path.exists(self.path): - return None - - layer = None - if self.layer_type == LayerType.RASTER: - layer = QgsRasterLayer(self.path, self.name) - - elif self.layer_type == LayerType.VECTOR: - layer = QgsVectorLayer(self.path, self.name) - - return layer - - def is_valid(self) -> bool: - """Checks if the corresponding map layer is valid. - - :returns: True if the map layer is valid, else False if map layer is - invalid or of None type. - :rtype: bool - """ - layer = self.to_map_layer() - if layer is None: - return False - - return layer.isValid() - - def __eq__(self, other) -> bool: - """Uses BaseModelComponent equality test rather than - what the dataclass default implementation will provide. - """ - return super().__eq__(other) - - -LayerModelComponentType = typing.TypeVar( - "LayerModelComponentType", bound=LayerModelComponent -) - - -class PriorityLayerType(IntEnum): - """Type of priority weighting layer.""" - - DEFAULT = 0 - NPV = 1 - - -@dataclasses.dataclass -class PriorityLayer(BaseModelComponent): - """Base class for model components storing priority weighting layers.""" - - groups: list - selected: bool = False - path: str = "" - type: PriorityLayerType = PriorityLayerType.DEFAULT - - -@dataclasses.dataclass -class NcsPathway(LayerModelComponent): - """Contains information about an NCS pathway layer.""" - - carbon_paths: typing.List[str] = dataclasses.field(default_factory=list) - - def __eq__(self, other: "NcsPathway") -> bool: - """Test equality of NcsPathway object with another - NcsPathway object using the attributes. - - Excludes testing the map layer for equality. - - :param other: NcsPathway object to compare with this object. - :type other: NcsPathway - - :returns: True if all the attribute values match, else False. - :rtype: bool - """ - base_equality = super().__eq__(other) - if not base_equality: - return False - - if self.path != other.path: - return False - - if self.layer_type != other.layer_type: - return False - - if self.user_defined != other.user_defined: - return False - - return True - - def add_carbon_path(self, carbon_path: str) -> bool: - """Add a carbon layer path. - - Checks if the path has already been defined or if it exists - in the file system. - - :returns: True if the carbon layer path was successfully - added, else False if the path has already been defined - or does not exist in the file system. - :rtype: bool - """ - if carbon_path in self.carbon_paths: - return False - - if not os.path.exists(carbon_path): - return False - - self.carbon_paths.append(carbon_path) - - return True - - def carbon_layers(self) -> typing.List[QgsRasterLayer]: - """Returns the list of carbon layers whose path is defined under - the :py:attr:`~carbon_paths` attribute. - - The caller should check the validity of the layers or use - :py:meth:`~is_carbon_valid` function. - - :returns: Carbon layers for the NCS pathway or an empty list - if the path is not defined. - :rtype: list - """ - return [QgsRasterLayer(carbon_path) for carbon_path in self.carbon_paths] - - def is_carbon_valid(self) -> bool: - """Checks if the carbon layers are valid. - - :returns: True if all carbon layers are valid, else False if - even one is invalid. If there are no carbon layers defined, it will - always return True. - :rtype: bool - """ - is_valid = True - for cl in self.carbon_layers(): - if not cl.isValid(): - is_valid = False - break - - return is_valid - - def is_valid(self) -> bool: - """Additional check to include validity of carbon layers.""" - valid = super().is_valid() - if not valid: - return False - - carbon_valid = self.is_carbon_valid() - if not carbon_valid: - return False - - return True - - -@dataclasses.dataclass -class Activity(LayerModelComponent): - """Contains information about an activity used in a scenario. - If the layer has been set then it will - not be possible to add NCS pathways unless the layer - is cleared. - Priority will be given to the layer property. - """ - - pathways: typing.List[NcsPathway] = dataclasses.field(default_factory=list) - priority_layers: typing.List[typing.Dict] = dataclasses.field(default_factory=list) - layer_styles: dict = dataclasses.field(default_factory=dict) - style_pixel_value: int = -1 - - def __post_init__(self): - """Pre-checks on initialization.""" - super().__post_init__() - - # Ensure there are no duplicate pathways. - uuids = [str(p.uuid) for p in self.pathways] - - if len(set(uuids)) != len(uuids): - msg = "Duplicate pathways found in activity" - raise ValueError(f"{msg} {self.name}.") - - # Reset pathways if layer has also been set. - if self.to_map_layer() is not None and len(self.pathways) > 0: - self.pathways = [] - - def contains_pathway(self, pathway_uuid: str) -> bool: - """Checks if there is an NCS pathway matching the given UUID. - - :param pathway_uuid: UUID to search for in the collection. - :type pathway_uuid: str - - :returns: True if there is a matching NCS pathway, else False. - :rtype: bool - """ - ncs_pathway = self.pathway_by_uuid(pathway_uuid) - if ncs_pathway is None: - return False - - return True - - def add_ncs_pathway(self, ncs: NcsPathway) -> bool: - """Adds an NCS pathway object to the collection. - - :param ncs: NCS pathway to be added to the activity. - :type ncs: NcsPathway - - :returns: True if the NCS pathway was successfully added, else False - if there was an existing NCS pathway object with a similar UUID or - the layer property had already been set. - """ - - if not ncs.is_valid(): - return False - - if self.contains_pathway(str(ncs.uuid)): - return False - - self.pathways.append(ncs) - - return True - - def clear_layer(self): - """Removes a reference to the layer URI defined in the path attribute.""" - self.path = "" - - def remove_ncs_pathway(self, pathway_uuid: str) -> bool: - """Removes the NCS pathway with a matching UUID from the collection. - - :param pathway_uuid: UUID for the NCS pathway to be removed. - :type pathway_uuid: str - - :returns: True if the NCS pathway object was successfully removed, - else False if there is no object matching the given UUID. - :rtype: bool - """ - idxs = [i for i, p in enumerate(self.pathways) if str(p.uuid) == pathway_uuid] - - if len(idxs) == 0: - return False - - rem_idx = idxs[0] - _ = self.pathways.pop(rem_idx) - - return True - - def pathway_by_uuid(self, pathway_uuid: str) -> typing.Union[NcsPathway, None]: - """Returns an NCS pathway matching the given UUID. - - :param pathway_uuid: UUID for the NCS pathway to retrieve. - :type pathway_uuid: str - - :returns: NCS pathway object matching the given UUID else None if - not found. - :rtype: NcsPathway - """ - pathways = [p for p in self.pathways if str(p.uuid) == pathway_uuid] - - if len(pathways) == 0: - return None - - return pathways[0] - - def pw_layers(self) -> typing.List[QgsRasterLayer]: - """Returns the list of priority weighting layers defined under - the :py:attr:`~priority_layers` attribute. - - :returns: Priority layers for the implementation or an empty list - if the path is not defined. - :rtype: list - """ - return [QgsRasterLayer(layer.get("path")) for layer in self.priority_layers] - - def is_pwls_valid(self) -> bool: - """Checks if the priority layers are valid. - - :returns: True if all priority layers are valid, else False if - even one is invalid. If there are no priority layers defined, it will - always return True. - :rtype: bool - """ - is_valid = True - for cl in self.pw_layers(): - if not cl.isValid(): - is_valid = False - break - - return is_valid - - def is_valid(self) -> bool: - """Includes an additional check to assert if NCS pathways have - been specified if the layer has not been set or is not valid. - - Does not check for validity of individual NCS pathways in the - collection. - """ - if self.to_map_layer() is not None: - return super().is_valid() - else: - if len(self.pathways) == 0: - return False - - if not self.is_pwls_valid(): - return False - - return True - - def scenario_layer_style_info(self) -> dict: - """Returns the fill symbol properties for styling the activity - layer in the final scenario result. - - :returns: Fill symbol properties for the activity layer - styling in the scenario layer or an empty dictionary if there was - no definition found in the root style. - :rtype: dict - """ - if ( - len(self.layer_styles) == 0 - or ACTIVITY_SCENARIO_STYLE_ATTRIBUTE not in self.layer_styles - ): - return dict() - - return self.layer_styles[ACTIVITY_SCENARIO_STYLE_ATTRIBUTE] - - def activity_layer_style_info(self) -> dict: - """Returns the color ramp properties for styling the activity - layer resulting from a scenario run. - - :returns: Color ramp properties for the activity styling or an - empty dictionary if there was no definition found in the root - style. - :rtype: dict - """ - if ( - len(self.layer_styles) == 0 - or ACTIVITY_LAYER_STYLE_ATTRIBUTE not in self.layer_styles - ): - return dict() - - return self.layer_styles[ACTIVITY_LAYER_STYLE_ATTRIBUTE] - - def scenario_fill_symbol(self) -> typing.Union[QgsFillSymbol, None]: - """Creates a fill symbol for the activity in the scenario. - - :returns: Fill symbol for the activity in the scenario - or None if there was no definition found. - :rtype: QgsFillSymbol - """ - scenario_style_info = self.scenario_layer_style_info() - if len(scenario_style_info) == 0: - return None - - return QgsFillSymbol.createSimple(scenario_style_info) - - def color_ramp(self) -> typing.Union[QgsColorRamp, None]: - """Create a color ramp for styling the activity layer resulting - from a scenario run. - - :returns: A color ramp for styling the activity layer or None - if there was no definition found. - :rtype: QgsColorRamp - """ - model_layer_info = self.activity_layer_style_info() - if len(model_layer_info) == 0: - return None - - ramp_info = model_layer_info.get(COLOR_RAMP_PROPERTIES_ATTRIBUTE, None) - if ramp_info is None or len(ramp_info) == 0: - return None - - ramp_type = model_layer_info.get(COLOR_RAMP_TYPE_ATTRIBUTE, None) - if ramp_type is None: - return None - - # New ramp types will need to be added here manually - if ramp_type == QgsColorBrewerColorRamp.typeString(): - return QgsColorBrewerColorRamp.create(ramp_info) - elif ramp_type == QgsCptCityColorRamp.typeString(): - return QgsCptCityColorRamp.create(ramp_info) - elif ramp_type == QgsGradientColorRamp.typeString(): - return QgsGradientColorRamp.create(ramp_info) - elif ramp_type == QgsLimitedRandomColorRamp.typeString(): - return QgsLimitedRandomColorRamp.create(ramp_info) - elif ramp_type == QgsPresetSchemeColorRamp.typeString(): - return QgsPresetSchemeColorRamp.create(ramp_info) - elif ramp_type == QgsRandomColorRamp.typeString(): - return QgsRandomColorRamp() - - return None - - -class ScenarioState(Enum): - """Defines scenario analysis process states""" - - IDLE = 0 - RUNNING = 1 - STOPPED = 3 - FINISHED = 4 - TERMINATED = 5 - - -@dataclasses.dataclass -class Scenario(BaseModelComponent): - """Object for the handling - workflow scenario information. - """ - - extent: SpatialExtent - activities: typing.List[Activity] - weighted_activities: typing.List[Activity] - priority_layer_groups: typing.List - state: ScenarioState = ScenarioState.IDLE - - -@dataclasses.dataclass -class ScenarioResult: - """Scenario result details.""" - - scenario: Scenario - created_date: datetime.datetime = datetime.datetime.now() - analysis_output: typing.Dict = None - output_layer_name: str = "" - scenario_directory: str = "" diff --git a/src/cplus_plugin/models/financial.py b/src/cplus_plugin/models/financial.py index 40f986eba..8f2c5881b 100644 --- a/src/cplus_plugin/models/financial.py +++ b/src/cplus_plugin/models/financial.py @@ -6,7 +6,7 @@ from enum import IntEnum import typing -from .base import Activity +from cplus_core.models.base import Activity @dataclasses.dataclass diff --git a/src/cplus_plugin/models/helpers.py b/src/cplus_plugin/models/helpers.py index 0400d0bb5..bf38b7332 100644 --- a/src/cplus_plugin/models/helpers.py +++ b/src/cplus_plugin/models/helpers.py @@ -14,7 +14,7 @@ QgsRectangle, ) -from .base import ( +from cplus_core.models.base import ( BaseModelComponent, BaseModelComponentType, Activity, diff --git a/src/cplus_plugin/models/report.py b/src/cplus_plugin/models/report.py index 8cf419598..85c16667c 100644 --- a/src/cplus_plugin/models/report.py +++ b/src/cplus_plugin/models/report.py @@ -8,7 +8,7 @@ from qgis.core import QgsFeedback, QgsRectangle -from .base import Scenario, ScenarioResult +from cplus_core.models.base import Scenario, ScenarioResult @dataclasses.dataclass diff --git a/src/cplus_plugin/models/validation.py b/src/cplus_plugin/models/validation.py index fe3d81b97..4c05fd45a 100644 --- a/src/cplus_plugin/models/validation.py +++ b/src/cplus_plugin/models/validation.py @@ -6,7 +6,7 @@ from enum import IntEnum import typing -from .base import ModelComponentType +from cplus_core.models.base import ModelComponentType class RuleType(IntEnum): diff --git a/test/model_data_for_testing.py b/test/model_data_for_testing.py index 5035caeb5..ca33950ff 100644 --- a/test/model_data_for_testing.py +++ b/test/model_data_for_testing.py @@ -20,7 +20,7 @@ UUID_ATTRIBUTE, ) from cplus_plugin.definitions.defaults import PILOT_AREA_EXTENT -from cplus_plugin.models.base import ( +from cplus_core.models.base import ( Activity, LayerType, NcsPathway, diff --git a/test/test_data_model_helpers.py b/test/test_data_model_helpers.py index 6f8c50589..bf389d3f0 100644 --- a/test/test_data_model_helpers.py +++ b/test/test_data_model_helpers.py @@ -5,7 +5,7 @@ from unittest import TestCase -from cplus_plugin.models.base import NcsPathway +from cplus_core.models.base import NcsPathway from cplus_plugin.models.helpers import ( clone_layer_component, create_ncs_pathway, diff --git a/test/test_scenario_tasks.py b/test/test_scenario_tasks.py index 62a7849d8..1e37fe03b 100644 --- a/test/test_scenario_tasks.py +++ b/test/test_scenario_tasks.py @@ -17,7 +17,7 @@ from cplus_plugin.conf import settings_manager, Settings from cplus_plugin.tasks import ScenarioAnalysisTask -from cplus_plugin.models.base import Scenario, NcsPathway, Activity +from cplus_core.models.base import Scenario, NcsPathway, Activity class ScenarioAnalysisTaskTest(unittest.TestCase): From fa8e352947d48fe79dfcf6c43c21f8de8bd0d6a5 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Thu, 12 Sep 2024 13:38:49 +0700 Subject: [PATCH 03/12] update version packaging lib --- requirements-dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 798cde711..09a3cda4b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,7 +32,8 @@ mkdocs-material == 8.1.7 mkdocstrings-python == 1.6.0 mkdocs-video == 1.1.0 mkdocs == 1.2.3 -packaging == 21.3 +# packaging == 21.3 +packaging == 24.0 pygments == 2.11.2 pymdown-extensions == 9.1 pyparsing == 3.0.6 From 5e4de1c5b13daeba3f22365b5385af73ed208c1f Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Thu, 12 Sep 2024 14:03:48 +0700 Subject: [PATCH 04/12] fix import and sampling method --- src/cplus_plugin/api/scenario_task_api_client.py | 6 ++++-- test/test_scenario_tasks.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cplus_plugin/api/scenario_task_api_client.py b/src/cplus_plugin/api/scenario_task_api_client.py index aaf28d917..169e9d3c0 100644 --- a/src/cplus_plugin/api/scenario_task_api_client.py +++ b/src/cplus_plugin/api/scenario_task_api_client.py @@ -495,8 +495,10 @@ def build_scenario_detail_json(self) -> None: snap_rescale = self.get_settings_value( Settings.RESCALE_VALUES, default=False, setting_type=bool ) - resampling_method = self.get_settings_value( - Settings.RESAMPLING_METHOD, default=0 + resampling_method = int( + self.get_settings_value( + Settings.RESAMPLING_METHOD, default=0, setting_type=int + ) ) ncs_with_carbon = self.get_settings_value( Settings.NCS_WITH_CARBON, default=False, setting_type=bool diff --git a/test/test_scenario_tasks.py b/test/test_scenario_tasks.py index 1e37fe03b..7b2aacd23 100644 --- a/test/test_scenario_tasks.py +++ b/test/test_scenario_tasks.py @@ -16,7 +16,7 @@ from cplus_plugin.conf import settings_manager, Settings -from cplus_plugin.tasks import ScenarioAnalysisTask +from cplus_core.analysis import ScenarioAnalysisTask from cplus_core.models.base import Scenario, NcsPathway, Activity From b4deb45c0f4d97496a3e0c609c6be16b25d3fa5d Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Fri, 13 Sep 2024 20:25:58 +0700 Subject: [PATCH 05/12] fix activities in task config --- src/cplus_plugin/gui/qgis_cplus_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cplus_plugin/gui/qgis_cplus_main.py b/src/cplus_plugin/gui/qgis_cplus_main.py index 0441a3f7d..4e833badb 100644 --- a/src/cplus_plugin/gui/qgis_cplus_main.py +++ b/src/cplus_plugin/gui/qgis_cplus_main.py @@ -1543,6 +1543,7 @@ def create_task_config(self, scenario: Scenario) -> TaskConfig: scenario, settings_manager.get_priority_layers(), scenario.priority_layer_groups, + scenario.activities, settings_manager.get_all_activities(), settings_manager.get_value( Settings.SNAPPING_ENABLED, default=False, setting_type=bool From cdbea7f7e5743de0a16ec4a620fe220c664a6e4a Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Tue, 17 Sep 2024 05:36:01 +0700 Subject: [PATCH 06/12] fix tests --- .../api/scenario_history_tasks.py | 44 ++-- src/cplus_plugin/gui/qgis_cplus_main.py | 14 +- test/test_scenario_history_tasks.py | 34 +-- test/test_scenario_tasks.py | 198 ++++++++---------- 4 files changed, 134 insertions(+), 156 deletions(-) diff --git a/src/cplus_plugin/api/scenario_history_tasks.py b/src/cplus_plugin/api/scenario_history_tasks.py index 6aa6b6a52..fd4fe352e 100644 --- a/src/cplus_plugin/api/scenario_history_tasks.py +++ b/src/cplus_plugin/api/scenario_history_tasks.py @@ -15,7 +15,7 @@ from .scenario_task_api_client import ScenarioAnalysisTaskApiClient from ..conf import settings_manager from cplus_core.models.base import Scenario -from cplus_core.models.base import SpatialExtent +from cplus_core.analysis import TaskConfig from ..utils import log @@ -133,38 +133,36 @@ class FetchScenarioOutputTask(ScenarioAnalysisTaskApiClient): def __init__( self, - analysis_scenario_name, - analysis_scenario_description, - analysis_activities, - analysis_priority_layers_groups, - analysis_extent, - scenario, - scenario_directory, + task_config: TaskConfig, + extent_box, ): - super(FetchScenarioOutputTask, self).__init__( - analysis_scenario_name, - analysis_scenario_description, - analysis_activities, - analysis_priority_layers_groups, - analysis_extent, - scenario, - SpatialExtent(scenario.extent.bbox), - ) + super(FetchScenarioOutputTask, self).__init__(task_config, extent_box) self.request = CplusApiRequest() self.status_pooling = None self.logs = [] self.total_file_output = 0 self.downloaded_output = 0 self.scenario_status = None - self.scenario_directory = scenario_directory - self.scenario_api_uuid = scenario.uuid - self.scenario = scenario + self.scenario_api_uuid = task_config.scenario.uuid + self.scenario = task_config.scenario self.scenario_directory = None self.processing_cancelled = False self.scenario_result = None self.output_list = None - self.created_datetime + self.created_datetime = None + + def _get_scenario_directory(self, created_datetime: datetime.datetime) -> str: + """Generate scenario directory for current task. + + :return: Path to scenario directory + :rtype: str + """ + base_dir = self.task_config.base_dir + return os.path.join( + f"{base_dir}", + "scenario_" f'{created_datetime.strftime("%Y_%m_%d_%H_%M_%S")}', + ) def run(self): """Execute the task logic. @@ -183,7 +181,9 @@ def run(self): self.created_datetime = datetime.datetime.strptime( self.new_scenario_detail["submitted_on"], "%Y-%m-%dT%H:%M:%SZ" ) - self.scenario_directory = self.get_scenario_directory() + self.scenario_directory = self._get_scenario_directory( + self.created_datetime + ) if os.path.exists(self.scenario_directory): for file in os.listdir(self.scenario_directory): if file != "processing.log": diff --git a/src/cplus_plugin/gui/qgis_cplus_main.py b/src/cplus_plugin/gui/qgis_cplus_main.py index b0a82b6ac..74c9c81b8 100644 --- a/src/cplus_plugin/gui/qgis_cplus_main.py +++ b/src/cplus_plugin/gui/qgis_cplus_main.py @@ -1303,15 +1303,11 @@ def load_scenario(self, scenario_identifier=None): ) progress_dialog.run_dialog() - analysis_task = FetchScenarioOutputTask( - self.analysis_scenario_name, - self.analysis_scenario_description, - self.analysis_activities, - self.analysis_priority_layers_groups, - self.analysis_extent, - scenario, - None, - ) + task_config = self.create_task_config(scenario) + task_config.all_activities = self.analysis_activities + task_config.base_dir = settings_manager.get_value(Settings.BASE_DIR) + + analysis_task = FetchScenarioOutputTask(task_config, scenario.extent) analysis_task.scenario_api_uuid = scenario.server_uuid analysis_task.task_finished.connect(self.update_scenario_list) diff --git a/test/test_scenario_history_tasks.py b/test/test_scenario_history_tasks.py index d6336cb5d..d13cd3a7b 100644 --- a/test/test_scenario_history_tasks.py +++ b/test/test_scenario_history_tasks.py @@ -1,3 +1,4 @@ +import os import unittest import uuid from unittest.mock import patch, MagicMock @@ -8,6 +9,7 @@ ) from cplus_plugin.api.request import CplusApiRequest from cplus_core.models.base import Scenario, SpatialExtent +from cplus_core.analysis import TaskConfig class TestFetchScenarioHistoryTask(unittest.TestCase): @@ -141,15 +143,17 @@ def test_run_success( analysis_extent = SpatialExtent(bbox=scenario.extent.bbox) analysis_activities = scenario.activities analysis_priority_layers_groups = scenario.priority_layer_groups - task = FetchScenarioOutputTask( - analysis_scenario_name, - analysis_scenario_description, - analysis_activities, - analysis_priority_layers_groups, - analysis_extent, + task_config = TaskConfig( scenario, - None, + [], + analysis_priority_layers_groups, + analysis_activities, + analysis_activities, + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=os.path.join("/tmp", str(scenario.uuid)), ) + task = FetchScenarioOutputTask(task_config, None) with patch.object( FetchScenarioOutputTask, "fetch_scenario_output" @@ -183,15 +187,17 @@ def test_run_failure(self, mock_log): analysis_extent = SpatialExtent(bbox=scenario.extent.bbox) analysis_activities = scenario.activities analysis_priority_layers_groups = scenario.priority_layer_groups - task = FetchScenarioOutputTask( - analysis_scenario_name, - analysis_scenario_description, - analysis_activities, - analysis_priority_layers_groups, - analysis_extent, + task_config = TaskConfig( scenario, - None, + [], + analysis_priority_layers_groups, + analysis_activities, + analysis_activities, + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=os.path.join("/tmp", str(scenario.uuid)), ) + task = FetchScenarioOutputTask(task_config, None) with patch.object( CplusApiRequest, diff --git a/test/test_scenario_tasks.py b/test/test_scenario_tasks.py index 7b2aacd23..d91b62bd2 100644 --- a/test/test_scenario_tasks.py +++ b/test/test_scenario_tasks.py @@ -14,10 +14,9 @@ from qgis.core import QgsRasterLayer -from cplus_plugin.conf import settings_manager, Settings - -from cplus_core.analysis import ScenarioAnalysisTask +from cplus_core.analysis import ScenarioAnalysisTask, TaskConfig from cplus_core.models.base import Scenario, NcsPathway, Activity +from cplus_core.utils.helper import BaseFileUtils class ScenarioAnalysisTaskTest(unittest.TestCase): @@ -66,38 +65,35 @@ def test_scenario_pathways_analysis(self): priority_layer_groups=[], ) - analysis_task = ScenarioAnalysisTask( - "test_scenario_pathways_analysis", - "test_scenario_pathways_analysis_description", - [test_activity], - [], - test_layer.extent(), - scenario, - ) - - extent_string = ( - f"{test_extent.xMinimum()},{test_extent.xMaximum()}," - f"{test_extent.yMinimum()},{test_extent.yMaximum()}" - f" [{test_layer.crs().authid()}]" - ) - base_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data", "pathways", ) - scenario_directory = os.path.join( f"{base_dir}", f'scenario_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' f"{str(uuid.uuid4())[:4]}", ) + BaseFileUtils.create_new_dir(scenario_directory) - analysis_task.scenario_directory = scenario_directory + task_config = TaskConfig( + scenario, + [], + [], + [test_activity], + [test_activity], + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=scenario_directory, + ) + analysis_task = ScenarioAnalysisTask(task_config) - settings_manager.set_value(Settings.BASE_DIR, base_dir) - settings_manager.set_value(Settings.PATHWAY_SUITABILITY_INDEX, 1.0) - settings_manager.set_value(Settings.CARBON_COEFFICIENT, 1.0) + extent_string = ( + f"{test_extent.xMinimum()},{test_extent.xMaximum()}," + f"{test_extent.yMinimum()},{test_extent.yMaximum()}" + f" [{test_layer.crs().authid()}]" + ) past_stat = test_layer.dataProvider().bandStatistics(1) @@ -161,39 +157,35 @@ def test_scenario_pathways_normalization(self): priority_layer_groups=[], ) - analysis_task = ScenarioAnalysisTask( - "test_scenario_pathways_normalization", - "test_scenario_pathways_normalization_description", - [test_activity], - [], - test_layer.extent(), - scenario, - ) - - extent_string = ( - f"{test_extent.xMinimum()},{test_extent.xMaximum()}," - f"{test_extent.yMinimum()},{test_extent.yMaximum()}" - f" [{test_layer.crs().authid()}]" - ) - base_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data", "pathways", ) - scenario_directory = os.path.join( f"{base_dir}", f'scenario_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' f"_{str(uuid.uuid4())[:4]}", ) + BaseFileUtils.create_new_dir(scenario_directory) - analysis_task.scenario_directory = scenario_directory - - settings_manager.set_value(Settings.BASE_DIR, base_dir) - settings_manager.set_value(Settings.PATHWAY_SUITABILITY_INDEX, 1.0) - settings_manager.set_value(Settings.CARBON_COEFFICIENT, 1.0) + task_config = TaskConfig( + scenario, + [], + [], + [test_activity], + [test_activity], + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=scenario_directory, + ) + analysis_task = ScenarioAnalysisTask(task_config) + extent_string = ( + f"{test_extent.xMinimum()},{test_extent.xMaximum()}," + f"{test_extent.yMinimum()},{test_extent.yMaximum()}" + f" [{test_layer.crs().authid()}]" + ) past_stat = test_layer.dataProvider().bandStatistics(1) self.assertEqual(past_stat.minimumValue, 1.0) @@ -269,38 +261,35 @@ def test_scenario_activities_creation(self): priority_layer_groups=[], ) - analysis_task = ScenarioAnalysisTask( - "test_scenario_activities_creation", - "test_scenario_activities_creation_description", - [test_activity], - [], - test_extent, - scenario, - ) - - extent_string = ( - f"{test_extent.xMinimum()},{test_extent.xMaximum()}," - f"{test_extent.yMinimum()},{test_extent.yMaximum()}" - f" [{first_test_layer.crs().authid()}]" - ) - base_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data", "pathways", ) - scenario_directory = os.path.join( f"{base_dir}", f'scenario_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' f"_{str(uuid.uuid4())[:4]}", ) + BaseFileUtils.create_new_dir(scenario_directory) - analysis_task.scenario_directory = scenario_directory + task_config = TaskConfig( + scenario, + [], + [], + [test_activity], + [test_activity], + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=scenario_directory, + ) + analysis_task = ScenarioAnalysisTask(task_config) - settings_manager.set_value(Settings.BASE_DIR, base_dir) - settings_manager.set_value(Settings.PATHWAY_SUITABILITY_INDEX, 1.0) - settings_manager.set_value(Settings.CARBON_COEFFICIENT, 1.0) + extent_string = ( + f"{test_extent.xMinimum()},{test_extent.xMaximum()}," + f"{test_extent.yMinimum()},{test_extent.yMaximum()}" + f" [{first_test_layer.crs().authid()}]" + ) first_layer_stat = first_test_layer.dataProvider().bandStatistics(1) second_layer_stat = second_test_layer.dataProvider().bandStatistics(1) @@ -440,38 +429,35 @@ def test_scenario_activities_normalization(self): priority_layer_groups=[], ) - analysis_task = ScenarioAnalysisTask( - "test_scenario_activities_creation", - "test_scenario_activities_creation_description", - [test_activity], - [], - test_extent, - scenario, - ) - - extent_string = ( - f"{test_extent.xMinimum()},{test_extent.xMaximum()}," - f"{test_extent.yMinimum()},{test_extent.yMaximum()}" - f" [{activity_layer.crs().authid()}]" - ) - base_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data", "activities", ) - scenario_directory = os.path.join( f"{base_dir}", f'scenario_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' f"_{str(uuid.uuid4())[:4]}", ) + BaseFileUtils.create_new_dir(scenario_directory) - analysis_task.scenario_directory = scenario_directory + task_config = TaskConfig( + scenario, + [], + [], + [test_activity], + [test_activity], + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=scenario_directory, + ) + analysis_task = ScenarioAnalysisTask(task_config) - settings_manager.set_value(Settings.BASE_DIR, base_dir) - settings_manager.set_value(Settings.PATHWAY_SUITABILITY_INDEX, 1.0) - settings_manager.set_value(Settings.CARBON_COEFFICIENT, 1.0) + extent_string = ( + f"{test_extent.xMinimum()},{test_extent.xMaximum()}," + f"{test_extent.yMinimum()},{test_extent.yMaximum()}" + f" [{activity_layer.crs().authid()}]" + ) first_layer_stat = activity_layer.dataProvider().bandStatistics(1) @@ -525,10 +511,6 @@ def test_scenario_activities_weighting(self): "groups": [test_priority_group], } - settings_manager.save_priority_group(test_priority_group) - - settings_manager.save_priority_layer(priority_layer_1) - test_activity = Activity( uuid=uuid.uuid4(), name="test_activity", @@ -538,8 +520,6 @@ def test_scenario_activities_weighting(self): priority_layers=[priority_layer_1], ) - settings_manager.save_activity(test_activity) - activity_layer = QgsRasterLayer(test_activity.path, test_activity.name) test_extent = activity_layer.extent() @@ -554,39 +534,35 @@ def test_scenario_activities_weighting(self): priority_layer_groups=[], ) - analysis_task = ScenarioAnalysisTask( - "test_scenario_activities_creation", - "test_scenario_activities_creation_description", - [test_activity], - [], - test_extent, - scenario, - ) - - extent_string = ( - f"{test_extent.xMinimum()},{test_extent.xMaximum()}," - f"{test_extent.yMinimum()},{test_extent.yMaximum()}" - f" [{activity_layer.crs().authid()}]" - ) - base_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data", "activities", ) - scenario_directory = os.path.join( f"{base_dir}", f'scenario_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' f"_{str(uuid.uuid4())[:4]}", ) + BaseFileUtils.create_new_dir(scenario_directory) - analysis_task.scenario_directory = scenario_directory - - settings_manager.set_value(Settings.BASE_DIR, base_dir) - settings_manager.set_value(Settings.PATHWAY_SUITABILITY_INDEX, 1.0) - settings_manager.set_value(Settings.CARBON_COEFFICIENT, 1.0) + task_config = TaskConfig( + scenario, + [priority_layer_1], + [test_priority_group], + [test_activity], + [test_activity], + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=scenario_directory, + ) + analysis_task = ScenarioAnalysisTask(task_config) + extent_string = ( + f"{test_extent.xMinimum()},{test_extent.xMaximum()}," + f"{test_extent.yMinimum()},{test_extent.yMaximum()}" + f" [{activity_layer.crs().authid()}]" + ) first_layer_stat = activity_layer.dataProvider().bandStatistics(1) self.assertEqual(first_layer_stat.minimumValue, 1.0) From 8e5a39f56c7d8ec41fe38e8891ef9e2ae4527d11 Mon Sep 17 00:00:00 2001 From: Zulfikar Akbar Muzakki Date: Mon, 7 Oct 2024 19:51:00 +0700 Subject: [PATCH 07/12] Update CPLUS core version --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 684fb77c4..aecbd5b63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ # cplus-core -# git+https://github.com/kartoza/cplus-core.git@v0.0.3 -git+https://github.com/kartoza/cplus-core.git@feat-refactor-conf \ No newline at end of file +git+https://github.com/kartoza/cplus-core.git@v0.0.4 \ No newline at end of file From b8328faa8ab45b8babf3a27c46a9a756d18f3f23 Mon Sep 17 00:00:00 2001 From: Zulfikar Akbar Muzakki Date: Wed, 20 Nov 2024 15:06:01 +0700 Subject: [PATCH 08/12] Use newest cplus-core release --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aecbd5b63..ffd7825b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ # cplus-core -git+https://github.com/kartoza/cplus-core.git@v0.0.4 \ No newline at end of file +git+https://github.com/kartoza/cplus-core.git@v0.0.5 \ No newline at end of file From 3d1ae35d021ddf982e40a2130156e34444ecd4f3 Mon Sep 17 00:00:00 2001 From: Zulfikar Akbar Muzakki Date: Mon, 25 Nov 2024 18:54:41 +0700 Subject: [PATCH 09/12] Fix masking test --- test/test_scenario_tasks.py | 48 +++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/test/test_scenario_tasks.py b/test/test_scenario_tasks.py index 15747dffd..df15f8dcf 100644 --- a/test/test_scenario_tasks.py +++ b/test/test_scenario_tasks.py @@ -7,7 +7,6 @@ import os import uuid -import processing import datetime from processing.core.Processing import Processing @@ -17,6 +16,7 @@ from cplus_core.analysis import ScenarioAnalysisTask, TaskConfig from cplus_core.models.base import Scenario, NcsPathway, Activity from cplus_core.utils.helper import BaseFileUtils +from cplus_plugin.conf import settings_manager, Settings class ScenarioAnalysisTaskTest(unittest.TestCase): @@ -594,6 +594,18 @@ def test_scenario_activities_weighting(self): self.assertEqual(stat.minimumValue, 5.0) self.assertEqual(stat.maximumValue, 27.0) + # coding=utf-8 + """Tests for the plugin processing tasks + + """ + + import unittest + + import uuid + import processing + + from processing.core.Processing import Processing + def test_scenario_activities_masking(self): activities_layer_directory = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data", "activities", "layers" @@ -632,16 +644,31 @@ def test_scenario_activities_masking(self): weighted_activities=[], priority_layer_groups=[], ) + base_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data", + "pathways", + ) + scenario_directory = os.path.join( + f"{base_dir}", + f'scenario_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' + f"{str(uuid.uuid4())[:4]}", + ) + BaseFileUtils.create_new_dir(scenario_directory) - analysis_task = ScenarioAnalysisTask( - "test_scenario_activities_masking", - "test_scenario_activities_masking_description", - [test_activity], - [], - test_extent, + task_config = TaskConfig( scenario, + [], + [], + [test_activity], + [test_activity], + pathway_suitability_index=1.0, + carbon_coefficient=1.0, + base_dir=scenario_directory, ) + analysis_task = ScenarioAnalysisTask(task_config) + extent_string = ( f"{test_extent.xMinimum()},{test_extent.xMaximum()}," f"{test_extent.yMinimum()},{test_extent.yMaximum()}" @@ -654,12 +681,6 @@ def test_scenario_activities_masking(self): "activities", ) - scenario_directory = os.path.join( - f"{base_dir}", - f'scenario_{datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")}' - f"_{str(uuid.uuid4())[:4]}", - ) - analysis_task.scenario_directory = scenario_directory settings_manager.set_value(Settings.BASE_DIR, base_dir) @@ -675,6 +696,7 @@ def test_scenario_activities_masking(self): [test_activity], extent_string, temporary_output=True ) + print(results) self.assertTrue(results) self.assertIsInstance(results, bool) From 6cf45b6ff42311fe96b492fb290cb5f3645c1b17 Mon Sep 17 00:00:00 2001 From: Zulfikar Akbar Muzakki Date: Mon, 25 Nov 2024 20:36:11 +0700 Subject: [PATCH 10/12] Update cplus-core version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ffd7825b3..650027233 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ # cplus-core -git+https://github.com/kartoza/cplus-core.git@v0.0.5 \ No newline at end of file +git+https://github.com/kartoza/cplus-core.git@v0.0.6 \ No newline at end of file From 8e4a1aed3f3376bc00b61dbf761bf7858b970078 Mon Sep 17 00:00:00 2001 From: Zulfikar Akbar Muzakki Date: Thu, 5 Dec 2024 09:44:36 +0700 Subject: [PATCH 11/12] Update cplus-core version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6bc0cabc2..e1433b3b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ # cplus-core -git+https://github.com/kartoza/cplus-core.git@v0.0.7 \ No newline at end of file +git+https://github.com/kartoza/cplus-core.git@v0.0.8 \ No newline at end of file From 7811138be6a106e99d58701e54e1a48a5d982893 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Tue, 15 Jul 2025 00:22:07 +0100 Subject: [PATCH 12/12] fix import core models --- src/cplus_plugin/gui/metrics_builder_dialog.py | 2 +- src/cplus_plugin/gui/metrics_builder_model.py | 2 +- src/cplus_plugin/gui/settings/cplus_options.py | 4 ++-- src/cplus_plugin/gui/settings/priority_layer_add.py | 2 +- src/cplus_plugin/lib/carbon.py | 3 ++- src/cplus_plugin/main.py | 2 +- test/test_activity_widget.py | 1 - test/test_api_request.py | 2 +- test/test_metrics.py | 2 +- test/test_settings.py | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/cplus_plugin/gui/metrics_builder_dialog.py b/src/cplus_plugin/gui/metrics_builder_dialog.py index 617efd5f0..c565afa47 100644 --- a/src/cplus_plugin/gui/metrics_builder_dialog.py +++ b/src/cplus_plugin/gui/metrics_builder_dialog.py @@ -36,7 +36,7 @@ MetricColumnListItem, MetricColumnListModel, ) -from ..models.base import Activity +from cplus_core.models.base import Activity from ..models.helpers import clone_activity, clone_metric_configuration_profile from ..models.report import ( ActivityColumnMetric, diff --git a/src/cplus_plugin/gui/metrics_builder_model.py b/src/cplus_plugin/gui/metrics_builder_model.py index aac2d54da..595607689 100644 --- a/src/cplus_plugin/gui/metrics_builder_model.py +++ b/src/cplus_plugin/gui/metrics_builder_model.py @@ -10,7 +10,7 @@ from ..definitions.constants import ACTIVITY_NAME -from ..models.base import Activity +from cplus_core.models.base import Activity from ..models.report import ActivityColumnMetric, MetricColumn, MetricType from ..utils import FileUtils, tr diff --git a/src/cplus_plugin/gui/settings/cplus_options.py b/src/cplus_plugin/gui/settings/cplus_options.py index 2426a7298..2d150ea79 100644 --- a/src/cplus_plugin/gui/settings/cplus_options.py +++ b/src/cplus_plugin/gui/settings/cplus_options.py @@ -47,7 +47,7 @@ settings_manager, Settings, ) -from ...models.base import DataSourceType +from ...models.source import DataSourceType from ...definitions.constants import CPLUS_OPTIONS_KEY, NO_DATA_VALUE from ...definitions.defaults import ( GENERAL_OPTIONS_TITLE, @@ -62,7 +62,7 @@ from ...lib.validation.feedback import ValidationFeedback from ...lib.validation.validators import DataValidator from ...models.validation import RuleInfo, RuleType -from ...models.base import DataSourceType, LayerModelComponent, LayerType +from cplus_core.models.base import LayerModelComponent, LayerType from ...trends_earth.constants import API_URL, TIMEOUT from ...utils import FileUtils, log, tr, convert_size from ...trends_earth import auth, api, download diff --git a/src/cplus_plugin/gui/settings/priority_layer_add.py b/src/cplus_plugin/gui/settings/priority_layer_add.py index aabce2942..4685d7f1d 100644 --- a/src/cplus_plugin/gui/settings/priority_layer_add.py +++ b/src/cplus_plugin/gui/settings/priority_layer_add.py @@ -10,7 +10,7 @@ ) from ...api.layer_tasks import CreateUpdateDefaultLayerTask from ...definitions.defaults import ICON_PATH, USER_DOCUMENTATION_SITE -from ...models.base import LayerType +from cplus_core.models.base import LayerType from ...trends_earth import api from ...trends_earth.constants import API_URL, TIMEOUT from ...utils import ( diff --git a/src/cplus_plugin/lib/carbon.py b/src/cplus_plugin/lib/carbon.py index 4f34c1485..c64b795b1 100644 --- a/src/cplus_plugin/lib/carbon.py +++ b/src/cplus_plugin/lib/carbon.py @@ -22,7 +22,8 @@ from qgis import processing from ..conf import settings_manager, Settings -from ..models.base import Activity, DataSourceType, NcsPathwayType +from cplus_core.models.base import Activity, NcsPathwayType +from ..models.source import DataSourceType from ..utils import log diff --git a/src/cplus_plugin/main.py b/src/cplus_plugin/main.py index de6b09dde..9835e9889 100644 --- a/src/cplus_plugin/main.py +++ b/src/cplus_plugin/main.py @@ -52,7 +52,7 @@ from .lib.reports.layout_items import CplusMapRepeatItemLayoutItemMetadata from .lib.reports.manager import report_manager from .lib.reports.metrics import register_metric_functions, unregister_metric_functions -from .models.base import PriorityLayerType +from cplus_core.models.base import PriorityLayerType from .models.report import MetricConfigurationProfile, MetricProfileCollection from .gui.settings.cplus_options import CplusOptionsFactory from .gui.settings.log_options import LogOptionsFactory diff --git a/test/test_activity_widget.py b/test/test_activity_widget.py index 9581f5956..0c460ff13 100644 --- a/test/test_activity_widget.py +++ b/test/test_activity_widget.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock from qgis.gui import QgsMessageBar from cplus_plugin.gui.activity_widget import ActivityContainerWidget -from cplus_plugin.models.base import Activity, NcsPathway from cplus_plugin.gui.component_item_model import ActivityItem, NcsPathwayItem from model_data_for_testing import get_activity, get_valid_ncs_pathway from utilities_for_testing import get_qgis_app diff --git a/test/test_api_request.py b/test/test_api_request.py index dd5d73b88..a96879264 100644 --- a/test/test_api_request.py +++ b/test/test_api_request.py @@ -20,7 +20,7 @@ from cplus_plugin.api.carbon import IrrecoverableCarbonDownloadTask from cplus_plugin.conf import settings_manager, Settings from cplus_plugin.definitions.defaults import BASE_API_URL, IRRECOVERABLE_CARBON_API_URL -from cplus_plugin.models.base import DataSourceType +from cplus_plugin.models.source import DataSourceType from utilities_for_testing import get_qgis_app diff --git a/test/test_metrics.py b/test/test_metrics.py index 955f0aa58..385ecc8f2 100644 --- a/test/test_metrics.py +++ b/test/test_metrics.py @@ -23,7 +23,7 @@ register_metric_functions, unregister_metric_functions, ) -from cplus_plugin.models.base import DataSourceType +from cplus_plugin.models.source import DataSourceType from cplus_plugin.models.helpers import create_metric_configuration from cplus_plugin.models.report import ActivityContextInfo diff --git a/test/test_settings.py b/test/test_settings.py index 766a72194..a926bcf3e 100644 --- a/test/test_settings.py +++ b/test/test_settings.py @@ -3,7 +3,7 @@ from utilities_for_testing import get_qgis_app from cplus_plugin.definitions.defaults import IRRECOVERABLE_CARBON_API_URL -from cplus_plugin.models.base import DataSourceType +from cplus_plugin.models.source import DataSourceType from cplus_plugin.gui.settings.cplus_options import CplusSettings from cplus_plugin.gui.settings.report_options import ReportSettingsWidget from cplus_plugin.conf import (