From 4afd956cd9f566df7076af9bc42a9882ee8c90a9 Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman <6911384+docxology@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:36:33 -0800 Subject: [PATCH 1/4] fix: integrate reviewed InferAnts local software contracts - Adopt the original A1-A9 and FS-1 software and regression paths at their accepted hashes. - Preserve explicit external-provider, scientific-corpus, runtime and historical evidence boundaries. --- .../Research/FieldSHIFT-2/FieldSHIFT-2.md | 186 +++---- .../IN_PROGRESS/FieldShift_TODO.md | 8 +- .../FieldSHIFT-2/IN_PROGRESS/README.md | 7 + .../Research/FieldSHIFT-2/IN_PROGRESS/SPEC.md | 7 + .../generate_pre_pro_shifted_grants.py | 28 +- .../IN_PROGRESS/language_analysis.py | 16 +- .../Meta_Analysis_Shifted_Dissertation.py | 487 ++++++++++-------- ...a_Analysis_Shifted_Dissertation_Outline.py | 471 +++++++++-------- .../Methods/Research/FieldSHIFT-2/README.md | 15 +- .../Methods/Research/FieldSHIFT-2/SPEC.md | 13 + .../dissertation_entity_evaluation.py | 19 +- .../dissertation_grant_evaluator.py | 19 +- .../generate_pro_shift_domains.py | 27 +- .../Research/FieldSHIFT-2/shift_domains.py | 21 +- .../Methods/Research/FieldSHIFT-2/utils.py | 29 +- 2_OPERATE/MetaInformAnt_Simulation.py | 15 +- 2_OPERATE/environment.py | 24 +- 2_OPERATE/execute_Simulation.py | 45 +- 2_OPERATE/plan_Simulation.py | 64 ++- 2_OPERATE/render_Simulation.py | 60 ++- 4_REPORT/AGENTS.md | 15 +- 4_REPORT/README.md | 16 +- 4_REPORT/execute_report.py | 2 +- 4_REPORT/specify_report.py | 8 +- 5_FOLLOWUP/AGENTS.md | 20 +- 5_FOLLOWUP/README.md | 26 +- 5_FOLLOWUP/execute_followup.py | 76 ++- 5_FOLLOWUP/specify_followup.py | 76 ++- 6_API/Knowledge_API.py | 60 +++ 6_API/MetaInformAnt_API.py | 53 +- README.md | 34 +- TODO.md | 17 +- docs/api/knowledge_api.md | 32 +- docs/api/metainformant_api.md | 15 + docs/guides/running_simulations.md | 20 +- docs/operations/simulation_operations.md | 23 +- docs/reference/configuration.md | 18 + gnn/gaussian_agent.gnn.md | 2 +- tests/source_contract_helpers.py | 83 +++ tests/test_executor_errors.py | 105 ++++ .../test_fieldshift_analysis_preservation.py | 273 ++++++++++ tests/test_fieldshift_output_preservation.py | 270 ++++++++++ tests/test_followup_contracts.py | 235 +++++++++ tests/test_grid_actions.py | 47 ++ tests/test_knowledge_api.py | 219 +++++++- tests/test_meta_api.py | 79 +++ tests/test_planner_lifecycle.py | 77 +++ tests/test_renderer.py | 64 +++ tests/test_report_contracts.py | 78 +++ 49 files changed, 2780 insertions(+), 824 deletions(-) create mode 100644 tests/source_contract_helpers.py create mode 100644 tests/test_executor_errors.py create mode 100644 tests/test_fieldshift_analysis_preservation.py create mode 100644 tests/test_fieldshift_output_preservation.py create mode 100644 tests/test_followup_contracts.py create mode 100644 tests/test_grid_actions.py create mode 100644 tests/test_planner_lifecycle.py create mode 100644 tests/test_renderer.py create mode 100644 tests/test_report_contracts.py diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/FieldSHIFT-2.md b/1_PREPARE/Methods/Research/FieldSHIFT-2/FieldSHIFT-2.md index 55eb7c84..81b783e0 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/FieldSHIFT-2.md +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/FieldSHIFT-2.md @@ -1,129 +1,69 @@ -# FieldSHIFT-2: Advanced Domain Shifting Framework +# FieldSHIFT-2: Domain Shifting Framework -FieldSHIFT-2 is a sophisticated system for executing domain shifting operations. It provides a high-level interface for users to transform input domains into shifted domains and generate corresponding dissertation outlines. +FieldSHIFT-2 contains research scripts for domain transformations, +dissertation/outline generation, evaluation, translation and analysis. +Real provider access and scientific-runtime qualification remain separate +from the existing-output preservation contract. -## Overview - -FieldSHIFT-2 is a framework for domain shifting operations. It processes input domains, generates pro-shifted and shifted domains, and creates dissertation outlines. The system also includes translation and linguistic analysis features for result evaluation. - -## Directory Structure +## Actual source layout +```text FieldSHIFT-2/ -├── src/ -│ ├── core/ -│ │ ├── FieldSHIFT-2.py -│ │ ├── config.py -│ │ └── utils.py -│ ├── modules/ -│ │ ├── domain_processing/ -│ │ │ ├── generate_pro_shift_domains.py -│ │ │ └── shift_domains.py -│ │ ├── dissertation/ -│ │ │ ├── dissertation_outline_generator.py -│ │ │ ├── dissertation_generator.py -│ │ │ ├── dissertation_improver.py -│ │ │ └── dissertation_translation.py -│ │ └── analysis/ -│ │ └── language_analysis.py -│ └── shared/ -│ └── constants.py -├── tests/ -│ └── [test files] -├── docs/ -│ └── [documentation files] +├── config.py +├── utils.py +├── generate_pro_shift_domains.py +├── shift_domains.py +├── dissertation_outline_generator.py +├── dissertation_generator.py +├── dissertation_explainer.py +├── dissertation_improver.py +├── dissertation_translation.py +├── dissertation_entity_evaluation.py +├── dissertation_grant_evaluator.py +├── Meta_Analysis_Shifted_Dissertation.py +├── Meta_Analysis_Shifted_Dissertation_Outline.py +├── IN_PROGRESS/ +│ ├── language_analysis.py +│ └── generate_pre_pro_shifted_grants.py ├── Inputs_and_Outputs/ -│ ├── Domain/ -│ ├── Pro-Shifted_Domain/ -│ ├── Shifted_Domain/ -│ ├── Shifted_Domain_Dissertation_Outline/ -│ └── Analyses/ -├── requirements.txt -├── README.md -└── .gitignore - -## Key Components - -### FieldSHIFT-2.py - -- **Input**: Configuration file (specified via --config argument) -- **Output**: Manages and orchestrates the entire domain shifting process, writing results to all designated output folders. -- **Functionality**: Acts as the main entry point for the FieldSHIFT-2 framework, coordinating the execution of various modules and ensuring the workflow progresses smoothly from input to final output. - -### generate_pro_shift_domains.py - -- **Input**: Domain files from `Inputs_and_Outputs/Domain/` -- **Output**: Pro-shifted domain files in `Inputs_and_Outputs/Pro-Shifted_Domain/` -- **Functionality**: Processes raw domain files to generate pro-shifted domains, which serve as intermediate representations before the final domain shift. This step involves initial transformations and preparations for more advanced domain shifting. - -### shift_domains.py - -- **Input**: Pro-shifted domain files from `Inputs_and_Outputs/Pro-Shifted_Domain/` -- **Output**: Shifted domain files in `Inputs_and_Outputs/Shifted_Domain/` -- **Functionality**: Executes the core domain shifting operations, transforming pro-shifted domains into their final shifted versions. This module leverages advanced language models to perform sophisticated domain transformations. - -### dissertation_outline_generator.py - -- **Input**: Shifted domain files from `Inputs_and_Outputs/Shifted_Domain/` -- **Output**: Dissertation outline files in `Inputs_and_Outputs/Shifted_Domain_Dissertation_Outline/` -- **Functionality**: Generates structured dissertation outlines based on the shifted domain files, ensuring coherence and logical flow. This module creates a high-level framework for dissertations, organizing content into a clear and logical structure. - -### dissertation_generator.py - -- **Input**: Dissertation outline files from `Inputs_and_Outputs/Shifted_Domain_Dissertation_Outline/` -- **Output**: Complete dissertation drafts in `Inputs_and_Outputs/Shifted_Domain_Dissertation_Outline/` -- **Functionality**: Expands dissertation outlines into full drafts, adding detailed content and ensuring academic rigor. This module enhances the outlines with comprehensive sections, including introduction, methodology, results, and discussion. - -### dissertation_improver.py - -- **Input**: Dissertation drafts from `Inputs_and_Outputs/Shifted_Domain_Dissertation_Outline/` -- **Output**: Improved dissertation drafts in `Inputs_and_Outputs/Shifted_Domain_Dissertation_Outline/` -- **Functionality**: Enhances the quality of dissertation drafts by refining content, improving clarity, and ensuring adherence to academic standards. This module focuses on polishing the drafts to meet high academic and publication standards. - -### dissertation_translation.py - -- **Input**: Dissertation drafts from `Inputs_and_Outputs/Shifted_Domain_Dissertation_Outline/` -- **Output**: Translated dissertation drafts in `Inputs_and_Outputs/Shifted_Domain_Dissertation_Outline/` -- **Functionality**: Translates dissertation drafts into different languages, maintaining the original meaning and context. This module ensures that the dissertations are accessible to a broader audience by providing accurate translations. - -### language_analysis.py - -- **Input**: All folders in `Inputs_and_Outputs/` -- **Output**: Analysis results in `Inputs_and_Outputs/Analyses/` -- **Functionality**: Performs comprehensive linguistic analysis on all input and output files, providing insights into language patterns, quality, and consistency. This module evaluates the linguistic aspects of the content, ensuring high-quality and consistent language use across all documents. - -### utils.py - -- **Functionality**: Provides utility functions for file I/O, logging, and LLM setup. This module contains helper functions to streamline file operations, manage logs, and configure language models, ensuring smooth operation across the entire project. It is used by all other modules to perform common tasks efficiently. - -### config.py - -- **Input**: Configuration file specified by command-line argument -- **Output**: Configuration dictionary used by other modules -- **Functionality**: Loads and parses the configuration file, providing a configuration dictionary that other modules use to access settings and parameters. This module ensures that all components of the framework are correctly configured and can access necessary parameters. - -## Installation - -1. Clone the repository - -## Usage - -Run the scripts. - -## Configuration - -Create a Python configuration file specifying: - -- Input/output directories -- Domain file paths -- Prompts -- LLM parameters -- Analysis parameters - -## Contributing - -Contribute on Github. -Contact - -## License +├── FieldSHIFT_Meta/ +└── FieldSHIFT-1/ +``` + +There is no `src/core/FieldSHIFT-2.py` master CLI or central `--config` +orchestrator. The scripts are flat modules except for the two named +`IN_PROGRESS` consumers. `config.py` supplies Python configuration; each +caller's actual directory/full-path arguments control its output, rather than +an invented common dissertation or analysis destination. +Preserved artifacts under `Inputs_and_Outputs/`, `FieldSHIFT_Meta/` and the +predecessor tree are not rewritten to match an aspirational layout. + +## Writers and execution boundary + +`save_file(content, full_path)` writes caller-serialized UTF-8 text; it has no +implicit directory/filename overload. Existing destinations, including empty +files, are preserved before artifact-specific input/provider/computation work. +Complete groups skip; partial groups preserve existing companions while +eligible missing outputs remain independent. Both meta-analysis modules +participate in this contract. + +`IN_PROGRESS/language_analysis.py` writes `analysis.json` and a conditional +PCA companion. Existing companions are preserved independently; nested +linguistic features are not silently flattened into a new scientific model. + +Normal imports and entrypoints retain their optional-library, key/configuration +and NLTK prerequisites. Preservation controls are not credential-free execution, +provider integration or permission to generate research outputs. Grant-product +generation and all-folder language analysis remain separate owner decisions; +see [the original pending requests](IN_PROGRESS/FieldShift_TODO.md). + +## References and maintenance + +The [pipeline inventory](README.md) and [writer specification](SPEC.md) describe +the current local interfaces. The predecessor paper and its historical CSVs +remain historical evidence, not newly executed writer entrypoints. + +Contribute through the repository's normal review process. +Contact . +Repository license: CC BY-NC-ND 4.0. -... diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/FieldShift_TODO.md b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/FieldShift_TODO.md index 6c85cd0e..4c155570 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/FieldShift_TODO.md +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/FieldShift_TODO.md @@ -1,4 +1,10 @@ -Check all methods that they skip files already there. +Existing-file preservation: the complete writer/caller ledger, both nested +scripts, both meta-analysis modules, empty-file preservation, partial groups +and offline no-provider-work cases passed the parent's governed focused +software acceptance on 2026-11-13. This closure is not based on only four +clustering files and is not credential/import-free entrypoint or scientific +runtime acceptance. The grant-generation and all-folder language-analysis +requests below remain separate and open (FS-2/FS-3). Write Pro-Grant (Completed catechism) for the Funding calls in the Grant Fond, for funding that PhD student. diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/README.md b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/README.md index 74ab1e6f..f96b3277 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/README.md +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/README.md @@ -9,6 +9,13 @@ IN PROGRESS — component of the FieldSHIFT-2 subsystem in the Active InferAnts - 📄 `generate_pre_pro_shifted_grants.py` - 📄 `language_analysis.py` +## Companion outputs + +Linguistic output comprises `analysis.json` and a conditional PCA image. +Existing companions are preserved independently, including empty files. +Default nested linguistic features are not silently flattened into a new +numeric feature model; scientific feature/runtime selection remains separate. + ## Related - Parent: [FieldSHIFT-2](../README.md) diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/SPEC.md b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/SPEC.md index 6d783b2f..fa23a59c 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/SPEC.md +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/SPEC.md @@ -14,6 +14,13 @@ IN PROGRESS — component of the FieldSHIFT-2 subsystem in the Active InferAnts - 📄 `generate_pre_pro_shifted_grants.py` - 📄 `language_analysis.py` +## Companion contract + +Linguistic output comprises `analysis.json` and a conditional PCA image. +Existing companions are preserved independently. Default nested linguistic +features are not silently flattened into a new numeric feature model; +scientific feature/runtime selection remains separate. + ## Version - **Framework**: Active InferAnts diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/generate_pre_pro_shifted_grants.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/generate_pre_pro_shifted_grants.py index 35fbe834..ef04bb43 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/generate_pre_pro_shifted_grants.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/generate_pre_pro_shifted_grants.py @@ -51,14 +51,18 @@ def generate_concatenated_domains(domains: List[str], prompt: str, output_dir: s for domain_a, domain_b in product(domains, repeat=2): domain_a_name = os.path.splitext(os.path.basename(domain_a))[0] domain_b_name = os.path.splitext(os.path.basename(domain_b))[0] + output_filename = f'concatenated_{domain_a_name}_to_{domain_b_name}.md' + output_path = os.path.join(output_dir, output_filename) + if os.path.exists(output_path): + logger.info(f"Output already exists; preserving {output_path}") + continue logger.info(f"Processing combination: {domain_a_name} and {domain_b_name}") domain_a_content = load_file(domain_a) domain_b_content = load_file(domain_b) concatenated_content = generate_pro_shift_domains(domain_a_content, domain_b_content, prompt) - output_filename = f'concatenated_{domain_a_name}_to_{domain_b_name}.md' - save_file(concatenated_content, output_dir, output_filename) + save_file(concatenated_content, output_path) logger.info(f"Concatenated domain saved as: {output_filename}") completed_combinations += 1 @@ -88,21 +92,25 @@ def process_domains(config: dict = DEFAULT_CONFIG) -> None: # Load domain contents domain_files = [os.path.join(input_domain_dir, f) for f in os.listdir(input_domain_dir) if f.endswith('.md')] - domains = [load_file(file) for file in domain_files] logger.info("Domains loaded successfully") logger.info("Starting concatenation process") - # Load the prompt - prompt_path = os.path.join(base_dir, 'Prompts', 'FieldSHIFT-2-Domain_Shift.md') - prompt = load_file(prompt_path) - - # Generate concatenated domains - generate_concatenated_domains(domain_files, prompt, config['output_dirs']['Concatenated_Domain']) + output_dir = config['output_dirs']['Concatenated_Domain'] + if any( + not os.path.exists(os.path.join( + output_dir, + f"concatenated_{os.path.splitext(os.path.basename(a))[0]}_to_{os.path.splitext(os.path.basename(b))[0]}.md", + )) + for a, b in product(domain_files, repeat=2) + ): + prompt_path = os.path.join(base_dir, 'Prompts', 'FieldSHIFT-2-Domain_Shift.md') + prompt = load_file(prompt_path) + generate_concatenated_domains(domain_files, prompt, output_dir) logger.info("Concatenation process completed successfully.") print(f"\n{'='*50}\nConcatenation process summary:\n{'-'*50}") - print(f"Total domains processed: {len(domains)}") + print(f"Total domains processed: {len(domain_files)}") print(f"Output directory: {config['output_dirs']['Concatenated_Domain']}") print(f"{'='*50}\n") except Exception as e: diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/language_analysis.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/language_analysis.py index 1061d607..72f7d640 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/language_analysis.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/IN_PROGRESS/language_analysis.py @@ -1,4 +1,5 @@ import os +import json import logging from typing import List, Dict, Any from textstat import textstat @@ -34,6 +35,9 @@ def calculate_readability(text: str) -> Dict[str, float]: } def perform_pca_analysis(features: List[float], output_dir: str, filename: str) -> None: + pca_plot_path = os.path.join(output_dir, f'{filename}_pca.png') + if os.path.exists(pca_plot_path): + return logger.info(f"Performing PCA analysis for {filename}") pca = PCA(n_components=2) principal_components = pca.fit_transform(features) @@ -44,7 +48,6 @@ def perform_pca_analysis(features: List[float], output_dir: str, filename: str) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') - pca_plot_path = os.path.join(output_dir, f'{filename}_pca.png') plt.savefig(pca_plot_path) logger.info(f"PCA plot saved to {pca_plot_path}") @@ -68,6 +71,10 @@ def analyze_text(text: str, analysis_params: Dict[str, Any]) -> Dict[str, Any]: return results def perform_linguistic_analysis(input_dir: str, output_dir: str) -> None: + analysis_path = os.path.join(output_dir, 'analysis.json') + pca_path = os.path.join(output_dir, f'{os.path.basename(input_dir)}_pca.png') + if os.path.exists(analysis_path) and os.path.exists(pca_path): + return analysis_params = DEFAULT_CONFIG['analysis_params'] results = {} all_features = [] @@ -83,10 +90,11 @@ def perform_linguistic_analysis(input_dir: str, output_dir: str) -> None: all_features.append(list(analysis_result.values())) os.makedirs(output_dir, exist_ok=True) - save_file(results, output_dir, 'analysis.json') - logger.info(f"Analysis results saved to {os.path.join(output_dir, 'analysis.json')}") + if not os.path.exists(analysis_path): + save_file(json.dumps(results, indent=2), analysis_path) + logger.info(f"Analysis results saved to {analysis_path}") - if all_features and all_features[0]: + if not os.path.exists(pca_path) and all_features and all_features[0]: perform_pca_analysis(all_features, output_dir, os.path.basename(input_dir)) def perform_meta_analysis(base_dir: str) -> None: diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation.py index 873e6f3e..525cb2f4 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation.py @@ -52,6 +52,51 @@ def __init__(self, input_dir='Inputs_and_Outputs/Shifted_Dissertations/', output self._ensure_output_dir_exists() logging.info("Initialization complete") + def _outputs_exist(self, paths): + return all(os.path.exists(os.path.join(self.output_dir, path)) for path in paths) + + def _pca_outputs(self): + return ( + 'PCA_Analysis/Top_Terms_Per_Component.png', + 'PCA_Analysis/Term_Loadings.png', + 'PCA_Analysis/Documents_And_Terms_In_PCA_Space.png', + 'PCA_Analysis/Explained_Variance.png', + 'PCA_Analysis/3D_PCA_Plot.png', + ) + + def _cluster_outputs(self): + return ( + 'Clustering/KMeans_Clustering_PCA.png', + 'Clustering/KMeans_Clustering_tSNE.png', + 'Clustering/KMeans_Clustering_SVD.png', + 'Clustering/Hierarchical_Clustering_PCA.png', + 'Projections/PCA_Projection_By_Entity.png', + 'Projections/PCA_Projection_By_Domain.png', + 'PCA_Analysis/Top_Terms_Per_Component_20.png', + 'Interactive/Interactive_3D_Scatter.html', + 'Dendrograms/Hierarchical_Clustering_Dendrogram.png', + ) + self._pca_outputs() + + def _tfidf_outputs(self): + return ( + 'TopTerms/Top_30_Terms_Bar_Chart.png', + 'Heatmaps/Term_Frequency_Heatmap.png', + 'Diversity/Shannon_Diversity_Indices.png', + 'Networks/Term_Network_Graph.png', + 'Heatmaps/Term_Correlation_Heatmap.png', + 'Heatmaps/Document_Similarity_Heatmap.png', + 'Heatmaps/Top_50_Terms_Document_Heatmap.png', + 'Heatmaps/Top_50_Terms_Cooccurrence_Heatmap.png', + ) + + def _analysis_outputs(self): + return ( + self._cluster_outputs() + + self._tfidf_outputs() + + ('Heatmaps/Entity_Domain_Combination_Heatmap.png',) + + tuple(f'WordClouds/{file}_wordcloud.png' for file in self.files) + ) + def _get_files(self): logging.info("Getting files from input directory") files = [f for f in os.listdir(self.input_dir) if f.endswith('.md')] @@ -108,50 +153,66 @@ def perform_tfidf_analysis(self): return top_terms, tfidf_matrix, vectorizer def cluster_documents(self, n_clusters=3): - logging.info("Clustering documents") + if self._outputs_exist(self._cluster_outputs()): + return + names = ( + 'KMeans_Clustering_PCA.png', + 'KMeans_Clustering_tSNE.png', + 'KMeans_Clustering_SVD.png', + 'Hierarchical_Clustering_PCA.png', + ) + missing = { + name: not self._outputs_exist((os.path.join('Clustering', name),)) + for name in names + } + projections = ( + 'Projections/PCA_Projection_By_Entity.png', + 'Projections/PCA_Projection_By_Domain.png', + ) + need_projections = not self._outputs_exist(projections) + need_interpretation = not self._outputs_exist(self._pca_outputs()) + need_terms = not self._outputs_exist(('PCA_Analysis/Top_Terms_Per_Component_20.png',)) + need_interactive = not self._outputs_exist(('Interactive/Interactive_3D_Scatter.html',)) + need_dendrogram = not self._outputs_exist(('Dendrograms/Hierarchical_Clustering_Dendrogram.png',)) _, tfidf_matrix, vectorizer = self.perform_tfidf_analysis() - - logging.info("Performing K-means clustering") - kmeans = KMeans(n_clusters=n_clusters, random_state=42) - kmeans_labels = kmeans.fit_predict(tfidf_matrix) - - logging.info("Performing Hierarchical clustering") - hierarchical = AgglomerativeClustering(n_clusters=n_clusters) - hierarchical_labels = hierarchical.fit_predict(tfidf_matrix.toarray()) - - logging.info("Performing dimensionality reduction") - pca = PCA(n_components=50, random_state=42) - tsne = TSNE(n_components=2, random_state=42, perplexity=5) - svd = TruncatedSVD(n_components=50, random_state=42) - - pca_coords = pca.fit_transform(tfidf_matrix.toarray())[:, :2] - tsne_coords = tsne.fit_transform(tfidf_matrix.toarray()) - svd_coords = svd.fit_transform(tfidf_matrix)[:, :2] - - # Plotting functions + kmeans_labels = None + hierarchical_labels = None + if any(missing[name] for name in names[:3]) or need_interactive: + kmeans = KMeans(n_clusters=n_clusters, random_state=42) + kmeans_labels = kmeans.fit_predict(tfidf_matrix) + if missing[names[3]]: + hierarchical = AgglomerativeClustering(n_clusters=n_clusters) + hierarchical_labels = hierarchical.fit_predict(tfidf_matrix.toarray()) + if missing[names[0]] or missing[names[3]] or need_projections or need_interpretation or need_terms or need_interactive: + pca = PCA(n_components=50, random_state=42) + pca_coordinates = pca.fit_transform(tfidf_matrix.toarray()) + pca_coords = pca_coordinates[:, :2] + if missing[names[1]]: + tsne = TSNE(n_components=2, random_state=42, perplexity=5) + tsne_coords = tsne.fit_transform(tfidf_matrix.toarray()) + if missing[names[2]]: + svd = TruncatedSVD(n_components=50, random_state=42) + svd_coords = svd.fit_transform(tfidf_matrix)[:, :2] + def plot_clusters(coords, labels, title, filename, method): + if self._outputs_exist((os.path.join('Clustering', filename),)): + return logging.info(f"Plotting {title}") plt.figure(figsize=(16, 12)) scatter = plt.scatter(coords[:, 0], coords[:, 1], c=labels, cmap='viridis', s=100) plt.colorbar(scatter, label='Cluster') - - # Add confidence intervals for label in set(labels): cluster_points = coords[labels == label] - confidence_ellipse(cluster_points[:, 0], cluster_points[:, 1], plt.gca(), n_std=2.0, + confidence_ellipse(cluster_points[:, 0], cluster_points[:, 1], plt.gca(), n_std=2.0, edgecolor=plt.cm.viridis(label / max(labels))) - - # Add labeled eigenvectors if method == 'PCA': for i, (comp, var) in enumerate(zip(pca.components_, pca.explained_variance_ratio_)): - plt.arrow(0, 0, comp[0]*var*20, comp[1]*var*20, color='r', alpha=0.5, + plt.arrow(0, 0, comp[0]*var*20, comp[1]*var*20, color='r', alpha=0.5, head_width=0.05, head_length=0.05) - plt.text(comp[0]*var*20, comp[1]*var*20, f'PC{i+1}\n({var:.2f})', + plt.text(comp[0]*var*20, comp[1]*var*20, f'PC{i+1}\n({var:.2f})', color='r', ha='center', va='center') - for i, file in enumerate(self.texts.keys()): plt.annotate(file, (coords[i, 0], coords[i, 1]), fontsize=8, alpha=0.7) - plt.title(title, fontsize=24, pad=20) plt.xlabel(f'{method} Component 1', fontsize=14) plt.ylabel(f'{method} Component 2', fontsize=14) @@ -159,39 +220,34 @@ def plot_clusters(coords, labels, title, filename, method): plt.savefig(os.path.join(self.output_dir, 'Clustering', filename), dpi=300, bbox_inches='tight') plt.close() logging.info(f"Plot saved: {os.path.join(self.output_dir, 'Clustering', filename)}") - - # Generate plots - plot_clusters(pca_coords, kmeans_labels, 'KMeans_Clustering_PCA', 'KMeans_Clustering_PCA.png', 'PCA') - plot_clusters(tsne_coords, kmeans_labels, 'KMeans_Clustering_tSNE', 'KMeans_Clustering_tSNE.png', 't-SNE') - plot_clusters(svd_coords, kmeans_labels, 'KMeans_Clustering_SVD', 'KMeans_Clustering_SVD.png', 'SVD') - plot_clusters(pca_coords, hierarchical_labels, 'Hierarchical_Clustering_PCA', 'Hierarchical_Clustering_PCA.png', 'PCA') - - # Silhouette analysis - logging.info("Performing silhouette analysis") - silhouette_kmeans = silhouette_score(tfidf_matrix, kmeans_labels) - silhouette_hierarchical = silhouette_score(tfidf_matrix.toarray(), hierarchical_labels) - - logging.info(f"K-means Silhouette Score: {silhouette_kmeans:.4f}") - logging.info(f"Hierarchical Clustering Silhouette Score: {silhouette_hierarchical:.4f}") - - # Visualization methods for dimensional projections - logging.info("Visualizing projections") - self.visualize_projections(pca_coords, 'PCA') - - # PCA interpretability methods - logging.info("Performing PCA interpretability analysis") - self.pca_interpretability(pca, vectorizer) - - # Plot top terms per PCA component - self.plot_top_terms_per_component(pca, vectorizer.get_feature_names_out()) - # Generate interactive 3D scatter plot - self.generate_interactive_3d_scatter(pca_coords, kmeans_labels) + if missing[names[0]]: + plot_clusters(pca_coords, kmeans_labels, 'KMeans_Clustering_PCA', names[0], 'PCA') + if missing[names[1]]: + plot_clusters(tsne_coords, kmeans_labels, 'KMeans_Clustering_tSNE', names[1], 't-SNE') + if missing[names[2]]: + plot_clusters(svd_coords, kmeans_labels, 'KMeans_Clustering_SVD', names[2], 'SVD') + if missing[names[3]]: + plot_clusters(pca_coords, hierarchical_labels, 'Hierarchical_Clustering_PCA', names[3], 'PCA') + if kmeans_labels is not None: + logging.info("K-means Silhouette Score: %.4f", silhouette_score(tfidf_matrix, kmeans_labels)) + if hierarchical_labels is not None: + logging.info("Hierarchical Clustering Silhouette Score: %.4f", silhouette_score(tfidf_matrix.toarray(), hierarchical_labels)) + if need_projections: + self.visualize_projections(pca_coords, 'PCA') + if need_interpretation: + self.pca_interpretability(pca, vectorizer) + if need_terms: + self.plot_top_terms_per_component(pca, vectorizer.get_feature_names_out()) + if need_interactive: + self.generate_interactive_3d_scatter(pca_coordinates[:, :3], kmeans_labels) + if need_dendrogram: + self.generate_dendrogram(tfidf_matrix.toarray()) - # Generate dendrogram - self.generate_dendrogram(tfidf_matrix.toarray()) def generate_interactive_3d_scatter(self, pca_coords, labels): + if self._outputs_exist(('Interactive/Interactive_3D_Scatter.html',)): + return logging.info("Generating interactive 3D scatter plot") fig = go.Figure(data=[go.Scatter3d( x=pca_coords[:, 0], @@ -220,6 +276,8 @@ def generate_interactive_3d_scatter(self, pca_coords, labels): logging.info(f"Interactive 3D scatter plot saved: {os.path.join(self.output_dir, 'Interactive', 'Interactive_3D_Scatter.html')}") def generate_dendrogram(self, data): + if self._outputs_exist(('Dendrograms/Hierarchical_Clustering_Dendrogram.png',)): + return logging.info("Generating dendrogram") linked = linkage(data, 'ward') @@ -238,6 +296,8 @@ def generate_dendrogram(self, data): logging.info(f"Dendrogram saved: {os.path.join(self.output_dir, 'Dendrograms', 'Hierarchical_Clustering_Dendrogram.png')}") def plot_top_terms_per_component(self, pca, feature_names, n_terms=20): + if self._outputs_exist((f'PCA_Analysis/Top_Terms_Per_Component_{n_terms}.png',)): + return logging.info("Plotting top terms per PCA component") fig, axes = plt.subplots(1, 2, figsize=(20, 10), sharey=True) @@ -263,11 +323,19 @@ def add_value_labels(self, ax, values, labels): ax.text(value, i, f'{value:.2f}', ha='center', va='center', color='white', fontsize=10, fontweight='bold', bbox=dict(facecolor='black', alpha=0.7)) def visualize_projections(self, coords, method): + if self._outputs_exist(( + f'Projections/{method}_Projection_By_Entity.png', + f'Projections/{method}_Projection_By_Domain.png', + )): + return logging.info(f"Visualizing {method} projections") entities = [file.split('_')[0] for file in self.files] domains = [file.split('_')[1].split('.md')[0] if len(file.split('_')) > 1 else 'Unknown' for file in self.files] def plot_projection(color_by, title): + filename = f"{method}_Projection_By_{title.split(' by ')[-1].replace(' ', '_')}.png" + if self._outputs_exist((os.path.join('Projections', filename),)): + return logging.info(f"Plotting {title}") plt.figure(figsize=(16, 12)) unique_items = list(set(color_by)) @@ -306,7 +374,6 @@ def plot_projection(color_by, title): plt.ylabel(f'{method} Component 2', fontsize=14) plt.tight_layout(pad=3) - filename = f"{method}_Projection_By_{title.split(' by ')[-1].replace(' ', '_')}.png" plt.savefig(os.path.join(self.output_dir, 'Projections', filename), dpi=300, bbox_inches='tight') plt.close() @@ -314,129 +381,118 @@ def plot_projection(color_by, title): plot_projection(domains, f'{method} Projection by Domain') def pca_interpretability(self, pca, vectorizer): - logging.info("Performing PCA interpretability analysis") - feature_names = vectorizer.get_feature_names_out() - - # Top terms for each principal component + paths = self._pca_outputs() + missing = [not self._outputs_exist((path,)) for path in paths] + if not any(missing): + return + if any(missing[:3]): + feature_names = vectorizer.get_feature_names_out() + def print_top_terms(component, feature_names, n=10): top_indices = np.argsort(component)[::-1][:n] - top_terms = [(feature_names[i], component[i]) for i in top_indices] - return top_terms - - # Create a figure to show top terms for each principal component - n_components = 20 # Number of components to show - n_top_terms = 40 # Number of top terms to show for each component - - plt.figure(figsize=(20, 4 * n_components), dpi=300) - for i, component in enumerate(pca.components_[:n_components]): - top_terms = print_top_terms(component, feature_names, n_top_terms) - terms = [term for term, _ in top_terms] - loadings = [loading for _, loading in top_terms] - - # Sort by absolute loading value - sorted_indices = np.argsort(np.abs(loadings))[::-1] - terms = [terms[i] for i in sorted_indices] - loadings = [loadings[i] for i in sorted_indices] - - ax = plt.subplot(n_components, 1, i + 1) - colors = sns.color_palette("husl", n_colors=n_top_terms) - bars = plt.barh(range(n_top_terms), loadings, align='center', color=colors) - plt.yticks(range(n_top_terms), terms, fontsize=12) - plt.xlabel('Loading', fontsize=14) - plt.title(f'Top Terms for PC{i+1}', fontsize=16) - - # Add value labels - for bar in bars: - width = bar.get_width() - plt.text(width, bar.get_y() + bar.get_height()/2, - f'{width:.2f}', va='center', fontsize=10) - - sns.despine(left=True, bottom=True) - plt.grid(axis='x', linestyle='--', alpha=0.7) + return [(feature_names[i], component[i]) for i in top_indices] + + if missing[0]: + n_components = 20 + n_top_terms = 40 + plt.figure(figsize=(20, 4 * n_components), dpi=300) + for i, component in enumerate(pca.components_[:n_components]): + top_terms = print_top_terms(component, feature_names, n_top_terms) + terms = [term for term, _ in top_terms] + loadings = [loading for _, loading in top_terms] + sorted_indices = np.argsort(np.abs(loadings))[::-1] + terms = [terms[i] for i in sorted_indices] + loadings = [loadings[i] for i in sorted_indices] + ax = plt.subplot(n_components, 1, i + 1) + colors = sns.color_palette("husl", n_colors=n_top_terms) + bars = plt.barh(range(n_top_terms), loadings, align='center', color=colors) + plt.yticks(range(n_top_terms), terms, fontsize=12) + plt.xlabel('Loading', fontsize=14) + plt.title(f'Top Terms for PC{i+1}', fontsize=16) + for bar in bars: + width = bar.get_width() + plt.text(width, bar.get_y() + bar.get_height()/2, + f'{width:.2f}', va='center', fontsize=10) + sns.despine(left=True, bottom=True) + plt.grid(axis='x', linestyle='--', alpha=0.7) + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[0]), dpi=300, bbox_inches='tight') + plt.close() + for i, component in enumerate(pca.components_): + logging.info(f"Top terms for PC{i+1}:") + logging.info(print_top_terms(component, feature_names)) + + if missing[1]: + logging.info("Visualizing term loadings on principal components") + plt.figure(figsize=(20, 10)) + terms = feature_names + x = pca.components_[0] + y = pca.components_[1] + plt.scatter(x, y) + plt.xlabel("PC1") + plt.ylabel("PC2") + for i, term in enumerate(terms): + plt.annotate(term, (x[i], y[i])) + plt.title("Term_Loadings_On_Principal_Components") + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[1]), dpi=300, bbox_inches='tight') + plt.close() - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Top_Terms_Per_Component.png'), - dpi=300, bbox_inches='tight') - plt.close() - for i, component in enumerate(pca.components_): - logging.info(f"Top terms for PC{i+1}:") - logging.info(print_top_terms(component, feature_names)) + if missing[2]: + logging.info("Visualizing documents in PCA space with term vectors") + pca_coords = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :2] + plt.figure(figsize=(20, 10)) + plt.scatter(pca_coords[:, 0], pca_coords[:, 1]) + for i, file in enumerate(self.texts.keys()): + plt.annotate(file, (pca_coords[i, 0], pca_coords[i, 1]), fontsize=8) + n_terms = 20 + terms = feature_names + x = pca.components_[0] + y = pca.components_[1] + indices = np.argsort(np.sqrt(x**2 + y**2))[-n_terms:] + for i in indices: + plt.arrow(0, 0, x[i]*5, y[i]*5, color='r', alpha=0.5, head_width=0.05, head_length=0.05) + plt.text(x[i]*5.2, y[i]*5.2, terms[i], color='r', ha='center', va='center') + plt.xlabel("PC1") + plt.ylabel("PC2") + plt.title("Documents_And_Terms_In_PCA_Space") + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[2]), dpi=300, bbox_inches='tight') + plt.close() - # Visualize term loadings on principal components - logging.info("Visualizing term loadings on principal components") - plt.figure(figsize=(20, 10)) - terms = feature_names - x = pca.components_[0] - y = pca.components_[1] - plt.scatter(x, y) - plt.xlabel("PC1") - plt.ylabel("PC2") - for i, term in enumerate(terms): - plt.annotate(term, (x[i], y[i])) - plt.title("Term_Loadings_On_Principal_Components") - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Term_Loadings.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"Term loadings plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', 'Term_Loadings.png')}") + if missing[3]: + logging.info("Plotting variance explained by each principal component") + plt.figure(figsize=(10, 6)) + plt.plot(np.cumsum(pca.explained_variance_ratio_), marker='o', linestyle='--') + plt.xlabel('Number of Components') + plt.ylabel('Cumulative Explained Variance') + plt.title('Explained_Variance_By_Number_Of_Principal_Components') + plt.grid(True) + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[3]), dpi=300, bbox_inches='tight') + plt.close() + + if missing[4]: + logging.info("Creating 3D PCA plot") + from mpl_toolkits.mplot3d import Axes3D + fig = plt.figure(figsize=(12, 8)) + ax = fig.add_subplot(111, projection='3d') + pca_coords_3d = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :3] + ax.scatter(pca_coords_3d[:, 0], pca_coords_3d[:, 1], pca_coords_3d[:, 2], c='b', marker='o') + for i, file in enumerate(self.texts.keys()): + ax.text(pca_coords_3d[i, 0], pca_coords_3d[i, 1], pca_coords_3d[i, 2], file, fontsize=8) + ax.set_xlabel('PC1') + ax.set_ylabel('PC2') + ax.set_zlabel('PC3') + ax.set_title('3D_PCA_Plot') + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[4]), dpi=300, bbox_inches='tight') + plt.close() - # Visualize documents in PCA space with term vectors - logging.info("Visualizing documents in PCA space with term vectors") - pca_coords = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :2] - plt.figure(figsize=(20, 10)) - plt.scatter(pca_coords[:, 0], pca_coords[:, 1]) - for i, file in enumerate(self.texts.keys()): - plt.annotate(file, (pca_coords[i, 0], pca_coords[i, 1]), fontsize=8) - - # Add term vectors - n_terms = 20 # Number of terms to show - terms = feature_names - x = pca.components_[0] - y = pca.components_[1] - indices = np.argsort(np.sqrt(x**2 + y**2))[-n_terms:] - for i in indices: - plt.arrow(0, 0, x[i]*5, y[i]*5, color='r', alpha=0.5, head_width=0.05, head_length=0.05) - plt.text(x[i]*5.2, y[i]*5.2, terms[i], color='r', ha='center', va='center') - - plt.xlabel("PC1") - plt.ylabel("PC2") - plt.title("Documents_And_Terms_In_PCA_Space") - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Documents_And_Terms_In_PCA_Space.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"Documents and terms in PCA space plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', 'Documents_And_Terms_In_PCA_Space.png')}") - - # Variance explained by each principal component - logging.info("Plotting variance explained by each principal component") - plt.figure(figsize=(10, 6)) - plt.plot(np.cumsum(pca.explained_variance_ratio_), marker='o', linestyle='--') - plt.xlabel('Number of Components') - plt.ylabel('Cumulative Explained Variance') - plt.title('Explained_Variance_By_Number_Of_Principal_Components') - plt.grid(True) - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Explained_Variance.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"Explained variance plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', 'Explained_Variance.png')}") - - # 3D PCA plot - logging.info("Creating 3D PCA plot") - from mpl_toolkits.mplot3d import Axes3D - fig = plt.figure(figsize=(12, 8)) - ax = fig.add_subplot(111, projection='3d') - pca_coords_3d = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :3] - ax.scatter(pca_coords_3d[:, 0], pca_coords_3d[:, 1], pca_coords_3d[:, 2], c='b', marker='o') - for i, file in enumerate(self.texts.keys()): - ax.text(pca_coords_3d[i, 0], pca_coords_3d[i, 1], pca_coords_3d[i, 2], file, fontsize=8) - ax.set_xlabel('PC1') - ax.set_ylabel('PC2') - ax.set_zlabel('PC3') - ax.set_title('3D_PCA_Plot') - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', '3D_PCA_Plot.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"3D PCA plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', '3D_PCA_Plot.png')}") def entity_domain_analysis(self): + if self._outputs_exist(('Heatmaps/Entity_Domain_Combination_Heatmap.png',)): + return logging.info("Analyzing entity-domain combinations") entities = set() from_domains = set() @@ -480,6 +536,8 @@ def entity_domain_analysis(self): def generate_word_clouds(self): logging.info("Generating word clouds") for file, text in self.texts.items(): + if self._outputs_exist((f'WordClouds/{file}_wordcloud.png',)): + continue wordcloud = WordCloud(stopwords=self.stop_words, background_color='white', width=800, height=400).generate(text) plt.figure(figsize=(10, 5)) plt.imshow(wordcloud, interpolation='bilinear') @@ -491,6 +549,8 @@ def generate_word_clouds(self): logging.info(f"Word cloud saved: {os.path.join(self.output_dir, 'WordClouds', f'{file}_wordcloud.png')}") def plot_top_terms_bar_chart(self, top_terms, num_terms=30): + if self._outputs_exist((f'TopTerms/Top_{num_terms}_Terms_Bar_Chart.png',)): + return logging.info(f"Plotting top {num_terms} terms bar chart") # Convert Series to list and flatten @@ -521,6 +581,8 @@ def plot_top_terms_bar_chart(self, top_terms, num_terms=30): logging.info(f"Top {num_terms} terms bar chart saved: {os.path.join(self.output_dir, 'TopTerms', f'Top_{num_terms}_Terms_Bar_Chart.png')}") def plot_term_frequency_heatmap(self, tfidf_matrix, feature_names): + if self._outputs_exist(('Heatmaps/Term_Frequency_Heatmap.png',)): + return logging.info("Plotting term frequency heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) plt.figure(figsize=(20, 10)) @@ -534,6 +596,8 @@ def plot_term_frequency_heatmap(self, tfidf_matrix, feature_names): logging.info(f"Term frequency heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', 'Term_Frequency_Heatmap.png')}") def calculate_diversity_indices(self, tfidf_matrix): + if self._outputs_exist(('Diversity/Shannon_Diversity_Indices.png',)): + return logging.info("Calculating diversity indices") df = pd.DataFrame(tfidf_matrix.toarray(), index=self.texts.keys()) @@ -563,6 +627,8 @@ def calculate_diversity_indices(self, tfidf_matrix): logging.info(f"Diversity indices plot saved: {os.path.join(self.output_dir, 'Diversity', 'Shannon_Diversity_Indices.png')}") def plot_term_network(self, tfidf_matrix, feature_names): + if self._outputs_exist(('Networks/Term_Network_Graph.png',)): + return logging.info("Plotting term network graph") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) G = nx.Graph() @@ -588,6 +654,8 @@ def plot_term_network(self, tfidf_matrix, feature_names): logging.info(f"Term network graph saved: {os.path.join(self.output_dir, 'Networks', 'Term_Network_Graph.png')}") def plot_term_correlation_heatmap(self, tfidf_matrix, feature_names): + if self._outputs_exist(('Heatmaps/Term_Correlation_Heatmap.png',)): + return logging.info("Plotting term correlation heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) @@ -604,6 +672,8 @@ def plot_term_correlation_heatmap(self, tfidf_matrix, feature_names): logging.info(f"Term correlation heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', 'Term_Correlation_Heatmap.png')}") def plot_document_similarity_heatmap(self, tfidf_matrix): + if self._outputs_exist(('Heatmaps/Document_Similarity_Heatmap.png',)): + return logging.info("Plotting document similarity heatmap") similarity_matrix = (tfidf_matrix * tfidf_matrix.T).toarray() @@ -617,6 +687,8 @@ def plot_document_similarity_heatmap(self, tfidf_matrix): logging.info(f"Document similarity heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', 'Document_Similarity_Heatmap.png')}") def plot_term_document_heatmap(self, tfidf_matrix, feature_names, top_n=50): + if self._outputs_exist((f'Heatmaps/Top_{top_n}_Terms_Document_Heatmap.png',)): + return logging.info(f"Plotting top {top_n} terms-document heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) @@ -637,6 +709,8 @@ def plot_term_document_heatmap(self, tfidf_matrix, feature_names, top_n=50): logging.info(f"Top {top_n} terms-document heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', f'Top_{top_n}_Terms_Document_Heatmap.png')}") def plot_term_cooccurrence_heatmap(self, tfidf_matrix, feature_names, top_n=50): + if self._outputs_exist((f'Heatmaps/Top_{top_n}_Terms_Cooccurrence_Heatmap.png',)): + return logging.info(f"Plotting top {top_n} terms co-occurrence heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) @@ -659,43 +733,32 @@ def plot_term_cooccurrence_heatmap(self, tfidf_matrix, feature_names, top_n=50): logging.info(f"Top {top_n} terms co-occurrence heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', f'Top_{top_n}_Terms_Cooccurrence_Heatmap.png')}") def run_analysis(self): - print("Performing TF-IDF analysis...") - top_terms, tfidf_matrix, vectorizer = self.perform_tfidf_analysis() - print("Top terms for each document:") - print(top_terms) - - print("Plotting top terms bar chart...") - self.plot_top_terms_bar_chart(top_terms, num_terms=30) - - print("Plotting term frequency heatmap...") - self.plot_term_frequency_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Calculating diversity indices...") - self.calculate_diversity_indices(tfidf_matrix) - - print("Clustering documents...") - self.cluster_documents() - - print("Analyzing entity-domain combinations...") - self.entity_domain_analysis() - - print("Generating word clouds...") - self.generate_word_clouds() - - print("Plotting term network graph...") - self.plot_term_network(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Plotting term correlation heatmap...") - self.plot_term_correlation_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Plotting document similarity heatmap...") - self.plot_document_similarity_heatmap(tfidf_matrix) - - print("Plotting top terms-document heatmap...") - self.plot_term_document_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Plotting top terms co-occurrence heatmap...") - self.plot_term_cooccurrence_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if self._outputs_exist(self._analysis_outputs()): + return + if not self._outputs_exist(self._tfidf_outputs()): + top_terms, tfidf_matrix, vectorizer = self.perform_tfidf_analysis() + if not self._outputs_exist(('TopTerms/Top_30_Terms_Bar_Chart.png',)): + self.plot_top_terms_bar_chart(top_terms, num_terms=30) + if not self._outputs_exist(('Heatmaps/Term_Frequency_Heatmap.png',)): + self.plot_term_frequency_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Diversity/Shannon_Diversity_Indices.png',)): + self.calculate_diversity_indices(tfidf_matrix) + if not self._outputs_exist(('Networks/Term_Network_Graph.png',)): + self.plot_term_network(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Heatmaps/Term_Correlation_Heatmap.png',)): + self.plot_term_correlation_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Heatmaps/Document_Similarity_Heatmap.png',)): + self.plot_document_similarity_heatmap(tfidf_matrix) + if not self._outputs_exist(('Heatmaps/Top_50_Terms_Document_Heatmap.png',)): + self.plot_term_document_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Heatmaps/Top_50_Terms_Cooccurrence_Heatmap.png',)): + self.plot_term_cooccurrence_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(self._cluster_outputs()): + self.cluster_documents() + if not self._outputs_exist(('Heatmaps/Entity_Domain_Combination_Heatmap.png',)): + self.entity_domain_analysis() + if not self._outputs_exist(tuple(f'WordClouds/{file}_wordcloud.png' for file in self.files)): + self.generate_word_clouds() def confidence_ellipse(x, y, ax, n_std=3.0, facecolor='none', **kwargs): """ diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation_Outline.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation_Outline.py index 34be1f84..6ef8b6d5 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation_Outline.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/Meta_Analysis_Shifted_Dissertation_Outline.py @@ -45,6 +45,49 @@ def __init__(self, input_dir='Inputs_and_Outputs/Shifted_Domain_Dissertation_Out self._ensure_output_dir_exists() logging.info("Initialization complete") + def _outputs_exist(self, paths): + return all(os.path.exists(os.path.join(self.output_dir, path)) for path in paths) + + def _pca_outputs(self): + return ( + 'PCA_Analysis/Top_Terms_Per_Component.png', + 'PCA_Analysis/Term_Loadings.png', + 'PCA_Analysis/Documents_And_Terms_In_PCA_Space.png', + 'PCA_Analysis/Explained_Variance.png', + 'PCA_Analysis/3D_PCA_Plot.png', + ) + + def _cluster_outputs(self): + return ( + 'Clustering/KMeans_Clustering_PCA.png', + 'Clustering/KMeans_Clustering_tSNE.png', + 'Clustering/KMeans_Clustering_SVD.png', + 'Clustering/Hierarchical_Clustering_PCA.png', + 'Projections/PCA_Projection_By_Entity.png', + 'Projections/PCA_Projection_By_Domain.png', + 'PCA_Analysis/Top_Terms_Per_Component_20.png', + ) + self._pca_outputs() + + def _tfidf_outputs(self): + return ( + 'TopTerms/Top_30_Terms_Bar_Chart.png', + 'Heatmaps/Term_Frequency_Heatmap.png', + 'Diversity/Shannon_Diversity_Indices.png', + 'Networks/Term_Network_Graph.png', + 'Heatmaps/Term_Correlation_Heatmap.png', + 'Heatmaps/Document_Similarity_Heatmap.png', + 'Heatmaps/Top_50_Terms_Document_Heatmap.png', + 'Heatmaps/Top_50_Terms_Cooccurrence_Heatmap.png', + ) + + def _analysis_outputs(self): + return ( + self._cluster_outputs() + + self._tfidf_outputs() + + ('Heatmaps/Entity_Domain_Combination_Heatmap.png',) + + tuple(f'WordClouds/{file}_wordcloud.png' for file in self.files) + ) + def _get_files(self): logging.info("Getting files from input directory") files = [f for f in os.listdir(self.input_dir) if f.endswith('.md')] @@ -98,50 +141,63 @@ def perform_tfidf_analysis(self): return top_terms, tfidf_matrix, vectorizer def cluster_documents(self, n_clusters=3): - logging.info("Clustering documents") + if self._outputs_exist(self._cluster_outputs()): + return + names = ( + 'KMeans_Clustering_PCA.png', + 'KMeans_Clustering_tSNE.png', + 'KMeans_Clustering_SVD.png', + 'Hierarchical_Clustering_PCA.png', + ) + missing = { + name: not self._outputs_exist((os.path.join('Clustering', name),)) + for name in names + } + projections = ( + 'Projections/PCA_Projection_By_Entity.png', + 'Projections/PCA_Projection_By_Domain.png', + ) + need_projections = not self._outputs_exist(projections) + need_interpretation = not self._outputs_exist(self._pca_outputs()) + need_terms = not self._outputs_exist(('PCA_Analysis/Top_Terms_Per_Component_20.png',)) _, tfidf_matrix, vectorizer = self.perform_tfidf_analysis() - - logging.info("Performing K-means clustering") - kmeans = KMeans(n_clusters=n_clusters, random_state=42) - kmeans_labels = kmeans.fit_predict(tfidf_matrix) - - logging.info("Performing Hierarchical clustering") - hierarchical = AgglomerativeClustering(n_clusters=n_clusters) - hierarchical_labels = hierarchical.fit_predict(tfidf_matrix.toarray()) - - logging.info("Performing dimensionality reduction") - pca = PCA(n_components=50, random_state=42) - tsne = TSNE(n_components=2, random_state=42, perplexity=5) - svd = TruncatedSVD(n_components=50, random_state=42) - - pca_coords = pca.fit_transform(tfidf_matrix.toarray())[:, :2] - tsne_coords = tsne.fit_transform(tfidf_matrix.toarray()) - svd_coords = svd.fit_transform(tfidf_matrix)[:, :2] - - # Plotting functions + kmeans_labels = None + hierarchical_labels = None + if any(missing[name] for name in names[:3]): + kmeans = KMeans(n_clusters=n_clusters, random_state=42) + kmeans_labels = kmeans.fit_predict(tfidf_matrix) + if missing[names[3]]: + hierarchical = AgglomerativeClustering(n_clusters=n_clusters) + hierarchical_labels = hierarchical.fit_predict(tfidf_matrix.toarray()) + if missing[names[0]] or missing[names[3]] or need_projections or need_interpretation or need_terms: + pca = PCA(n_components=50, random_state=42) + pca_coords = pca.fit_transform(tfidf_matrix.toarray())[:, :2] + if missing[names[1]]: + tsne = TSNE(n_components=2, random_state=42, perplexity=5) + tsne_coords = tsne.fit_transform(tfidf_matrix.toarray()) + if missing[names[2]]: + svd = TruncatedSVD(n_components=50, random_state=42) + svd_coords = svd.fit_transform(tfidf_matrix)[:, :2] + def plot_clusters(coords, labels, title, filename, method): + if self._outputs_exist((os.path.join('Clustering', filename),)): + return logging.info(f"Plotting {title}") plt.figure(figsize=(16, 12)) scatter = plt.scatter(coords[:, 0], coords[:, 1], c=labels, cmap='viridis', s=100) plt.colorbar(scatter, label='Cluster') - - # Add confidence intervals for label in set(labels): cluster_points = coords[labels == label] - confidence_ellipse(cluster_points[:, 0], cluster_points[:, 1], plt.gca(), n_std=2.0, + confidence_ellipse(cluster_points[:, 0], cluster_points[:, 1], plt.gca(), n_std=2.0, edgecolor=plt.cm.viridis(label / max(labels))) - - # Add labeled eigenvectors if method == 'PCA': for i, (comp, var) in enumerate(zip(pca.components_, pca.explained_variance_ratio_)): - plt.arrow(0, 0, comp[0]*var*20, comp[1]*var*20, color='r', alpha=0.5, + plt.arrow(0, 0, comp[0]*var*20, comp[1]*var*20, color='r', alpha=0.5, head_width=0.05, head_length=0.05) - plt.text(comp[0]*var*20, comp[1]*var*20, f'PC{i+1}\n({var:.2f})', + plt.text(comp[0]*var*20, comp[1]*var*20, f'PC{i+1}\n({var:.2f})', color='r', ha='center', va='center') - for i, file in enumerate(self.texts.keys()): plt.annotate(file, (coords[i, 0], coords[i, 1]), fontsize=8, alpha=0.7) - plt.title(title, fontsize=24, pad=20) plt.xlabel(f'{method} Component 1', fontsize=14) plt.ylabel(f'{method} Component 2', fontsize=14) @@ -149,34 +205,32 @@ def plot_clusters(coords, labels, title, filename, method): plt.savefig(os.path.join(self.output_dir, 'Clustering', filename), dpi=300, bbox_inches='tight') plt.close() logging.info(f"Plot saved: {os.path.join(self.output_dir, 'Clustering', filename)}") - - # Generate plots - plot_clusters(pca_coords, kmeans_labels, 'KMeans_Clustering_PCA', 'KMeans_Clustering_PCA.png', 'PCA') - plot_clusters(tsne_coords, kmeans_labels, 'KMeans_Clustering_tSNE', 'KMeans_Clustering_tSNE.png', 't-SNE') - plot_clusters(svd_coords, kmeans_labels, 'KMeans_Clustering_SVD', 'KMeans_Clustering_SVD.png', 'SVD') - plot_clusters(pca_coords, hierarchical_labels, 'Hierarchical_Clustering_PCA', 'Hierarchical_Clustering_PCA.png', 'PCA') - - # Silhouette analysis - logging.info("Performing silhouette analysis") - from sklearn.metrics import silhouette_score - silhouette_kmeans = silhouette_score(tfidf_matrix, kmeans_labels) - silhouette_hierarchical = silhouette_score(tfidf_matrix.toarray(), hierarchical_labels) - - logging.info(f"K-means Silhouette Score: {silhouette_kmeans:.4f}") - logging.info(f"Hierarchical Clustering Silhouette Score: {silhouette_hierarchical:.4f}") - # Visualization methods for dimensional projections - logging.info("Visualizing projections") - self.visualize_projections(pca_coords, 'PCA') + if missing[names[0]]: + plot_clusters(pca_coords, kmeans_labels, 'KMeans_Clustering_PCA', names[0], 'PCA') + if missing[names[1]]: + plot_clusters(tsne_coords, kmeans_labels, 'KMeans_Clustering_tSNE', names[1], 't-SNE') + if missing[names[2]]: + plot_clusters(svd_coords, kmeans_labels, 'KMeans_Clustering_SVD', names[2], 'SVD') + if missing[names[3]]: + plot_clusters(pca_coords, hierarchical_labels, 'Hierarchical_Clustering_PCA', names[3], 'PCA') + if kmeans_labels is not None or hierarchical_labels is not None: + from sklearn.metrics import silhouette_score + if kmeans_labels is not None: + logging.info("K-means Silhouette Score: %.4f", silhouette_score(tfidf_matrix, kmeans_labels)) + if hierarchical_labels is not None: + logging.info("Hierarchical Clustering Silhouette Score: %.4f", silhouette_score(tfidf_matrix.toarray(), hierarchical_labels)) + if need_projections: + self.visualize_projections(pca_coords, 'PCA') + if need_interpretation: + self.pca_interpretability(pca, vectorizer) + if need_terms: + self.plot_top_terms_per_component(pca, vectorizer.get_feature_names_out()) - # PCA interpretability methods - logging.info("Performing PCA interpretability analysis") - self.pca_interpretability(pca, vectorizer) - - # Plot top terms per PCA component - self.plot_top_terms_per_component(pca, vectorizer.get_feature_names_out()) def plot_top_terms_per_component(self, pca, feature_names, n_terms=20): + if self._outputs_exist((f'PCA_Analysis/Top_Terms_Per_Component_{n_terms}.png',)): + return logging.info("Plotting top terms per PCA component") fig, axes = plt.subplots(1, 2, figsize=(20, 10), sharey=True) @@ -202,11 +256,19 @@ def add_value_labels(self, ax, values, labels): ax.text(value, i, f'{value:.2f}', ha='center', va='center', color='white', fontsize=10, fontweight='bold', bbox=dict(facecolor='black', alpha=0.7)) def visualize_projections(self, coords, method): + if self._outputs_exist(( + f'Projections/{method}_Projection_By_Entity.png', + f'Projections/{method}_Projection_By_Domain.png', + )): + return logging.info(f"Visualizing {method} projections") entities = [file.split('_')[0] for file in self.files] domains = [file.split('_')[1].split('.md')[0] if len(file.split('_')) > 1 else 'Unknown' for file in self.files] def plot_projection(color_by, title): + filename = f"{method}_Projection_By_{title.split(' by ')[-1].replace(' ', '_')}.png" + if self._outputs_exist((os.path.join('Projections', filename),)): + return logging.info(f"Plotting {title}") plt.figure(figsize=(16, 12)) unique_items = list(set(color_by)) @@ -245,7 +307,6 @@ def plot_projection(color_by, title): plt.ylabel(f'{method} Component 2', fontsize=14) plt.tight_layout(pad=3) - filename = f"{method}_Projection_By_{title.split(' by ')[-1].replace(' ', '_')}.png" plt.savefig(os.path.join(self.output_dir, 'Projections', filename), dpi=300, bbox_inches='tight') plt.close() @@ -253,129 +314,118 @@ def plot_projection(color_by, title): plot_projection(domains, f'{method} Projection by Domain') def pca_interpretability(self, pca, vectorizer): - logging.info("Performing PCA interpretability analysis") - feature_names = vectorizer.get_feature_names_out() - - # Top terms for each principal component + paths = self._pca_outputs() + missing = [not self._outputs_exist((path,)) for path in paths] + if not any(missing): + return + if any(missing[:3]): + feature_names = vectorizer.get_feature_names_out() + def print_top_terms(component, feature_names, n=10): top_indices = np.argsort(component)[::-1][:n] - top_terms = [(feature_names[i], component[i]) for i in top_indices] - return top_terms - - # Create a figure to show top terms for each principal component - n_components = 20 # Number of components to show - n_top_terms = 40 # Number of top terms to show for each component - - plt.figure(figsize=(20, 4 * n_components), dpi=300) - for i, component in enumerate(pca.components_[:n_components]): - top_terms = print_top_terms(component, feature_names, n_top_terms) - terms = [term for term, _ in top_terms] - loadings = [loading for _, loading in top_terms] - - # Sort by absolute loading value - sorted_indices = np.argsort(np.abs(loadings))[::-1] - terms = [terms[i] for i in sorted_indices] - loadings = [loadings[i] for i in sorted_indices] - - ax = plt.subplot(n_components, 1, i + 1) - colors = sns.color_palette("husl", n_colors=n_top_terms) - bars = plt.barh(range(n_top_terms), loadings, align='center', color=colors) - plt.yticks(range(n_top_terms), terms, fontsize=12) - plt.xlabel('Loading', fontsize=14) - plt.title(f'Top Terms for PC{i+1}', fontsize=16) - - # Add value labels - for bar in bars: - width = bar.get_width() - plt.text(width, bar.get_y() + bar.get_height()/2, - f'{width:.2f}', va='center', fontsize=10) - - sns.despine(left=True, bottom=True) - plt.grid(axis='x', linestyle='--', alpha=0.7) + return [(feature_names[i], component[i]) for i in top_indices] + + if missing[0]: + n_components = 20 + n_top_terms = 40 + plt.figure(figsize=(20, 4 * n_components), dpi=300) + for i, component in enumerate(pca.components_[:n_components]): + top_terms = print_top_terms(component, feature_names, n_top_terms) + terms = [term for term, _ in top_terms] + loadings = [loading for _, loading in top_terms] + sorted_indices = np.argsort(np.abs(loadings))[::-1] + terms = [terms[i] for i in sorted_indices] + loadings = [loadings[i] for i in sorted_indices] + ax = plt.subplot(n_components, 1, i + 1) + colors = sns.color_palette("husl", n_colors=n_top_terms) + bars = plt.barh(range(n_top_terms), loadings, align='center', color=colors) + plt.yticks(range(n_top_terms), terms, fontsize=12) + plt.xlabel('Loading', fontsize=14) + plt.title(f'Top Terms for PC{i+1}', fontsize=16) + for bar in bars: + width = bar.get_width() + plt.text(width, bar.get_y() + bar.get_height()/2, + f'{width:.2f}', va='center', fontsize=10) + sns.despine(left=True, bottom=True) + plt.grid(axis='x', linestyle='--', alpha=0.7) + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[0]), dpi=300, bbox_inches='tight') + plt.close() + for i, component in enumerate(pca.components_): + logging.info(f"Top terms for PC{i+1}:") + logging.info(print_top_terms(component, feature_names)) + + if missing[1]: + logging.info("Visualizing term loadings on principal components") + plt.figure(figsize=(20, 10)) + terms = feature_names + x = pca.components_[0] + y = pca.components_[1] + plt.scatter(x, y) + plt.xlabel("PC1") + plt.ylabel("PC2") + for i, term in enumerate(terms): + plt.annotate(term, (x[i], y[i])) + plt.title("Term_Loadings_On_Principal_Components") + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[1]), dpi=300, bbox_inches='tight') + plt.close() - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Top_Terms_Per_Component.png'), - dpi=300, bbox_inches='tight') - plt.close() - for i, component in enumerate(pca.components_): - logging.info(f"Top terms for PC{i+1}:") - logging.info(print_top_terms(component, feature_names)) + if missing[2]: + logging.info("Visualizing documents in PCA space with term vectors") + pca_coords = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :2] + plt.figure(figsize=(20, 10)) + plt.scatter(pca_coords[:, 0], pca_coords[:, 1]) + for i, file in enumerate(self.texts.keys()): + plt.annotate(file, (pca_coords[i, 0], pca_coords[i, 1]), fontsize=8) + n_terms = 20 + terms = feature_names + x = pca.components_[0] + y = pca.components_[1] + indices = np.argsort(np.sqrt(x**2 + y**2))[-n_terms:] + for i in indices: + plt.arrow(0, 0, x[i]*5, y[i]*5, color='r', alpha=0.5, head_width=0.05, head_length=0.05) + plt.text(x[i]*5.2, y[i]*5.2, terms[i], color='r', ha='center', va='center') + plt.xlabel("PC1") + plt.ylabel("PC2") + plt.title("Documents_And_Terms_In_PCA_Space") + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[2]), dpi=300, bbox_inches='tight') + plt.close() - # Visualize term loadings on principal components - logging.info("Visualizing term loadings on principal components") - plt.figure(figsize=(20, 10)) - terms = feature_names - x = pca.components_[0] - y = pca.components_[1] - plt.scatter(x, y) - plt.xlabel("PC1") - plt.ylabel("PC2") - for i, term in enumerate(terms): - plt.annotate(term, (x[i], y[i])) - plt.title("Term_Loadings_On_Principal_Components") - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Term_Loadings.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"Term loadings plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', 'Term_Loadings.png')}") + if missing[3]: + logging.info("Plotting variance explained by each principal component") + plt.figure(figsize=(10, 6)) + plt.plot(np.cumsum(pca.explained_variance_ratio_), marker='o', linestyle='--') + plt.xlabel('Number of Components') + plt.ylabel('Cumulative Explained Variance') + plt.title('Explained_Variance_By_Number_Of_Principal_Components') + plt.grid(True) + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[3]), dpi=300, bbox_inches='tight') + plt.close() + + if missing[4]: + logging.info("Creating 3D PCA plot") + from mpl_toolkits.mplot3d import Axes3D + fig = plt.figure(figsize=(12, 8)) + ax = fig.add_subplot(111, projection='3d') + pca_coords_3d = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :3] + ax.scatter(pca_coords_3d[:, 0], pca_coords_3d[:, 1], pca_coords_3d[:, 2], c='b', marker='o') + for i, file in enumerate(self.texts.keys()): + ax.text(pca_coords_3d[i, 0], pca_coords_3d[i, 1], pca_coords_3d[i, 2], file, fontsize=8) + ax.set_xlabel('PC1') + ax.set_ylabel('PC2') + ax.set_zlabel('PC3') + ax.set_title('3D_PCA_Plot') + plt.tight_layout() + plt.savefig(os.path.join(self.output_dir, paths[4]), dpi=300, bbox_inches='tight') + plt.close() - # Visualize documents in PCA space with term vectors - logging.info("Visualizing documents in PCA space with term vectors") - pca_coords = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :2] - plt.figure(figsize=(20, 10)) - plt.scatter(pca_coords[:, 0], pca_coords[:, 1]) - for i, file in enumerate(self.texts.keys()): - plt.annotate(file, (pca_coords[i, 0], pca_coords[i, 1]), fontsize=8) - - # Add term vectors - n_terms = 20 # Number of terms to show - terms = feature_names - x = pca.components_[0] - y = pca.components_[1] - indices = np.argsort(np.sqrt(x**2 + y**2))[-n_terms:] - for i in indices: - plt.arrow(0, 0, x[i]*5, y[i]*5, color='r', alpha=0.5, head_width=0.05, head_length=0.05) - plt.text(x[i]*5.2, y[i]*5.2, terms[i], color='r', ha='center', va='center') - - plt.xlabel("PC1") - plt.ylabel("PC2") - plt.title("Documents_And_Terms_In_PCA_Space") - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Documents_And_Terms_In_PCA_Space.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"Documents and terms in PCA space plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', 'Documents_And_Terms_In_PCA_Space.png')}") - - # Variance explained by each principal component - logging.info("Plotting variance explained by each principal component") - plt.figure(figsize=(10, 6)) - plt.plot(np.cumsum(pca.explained_variance_ratio_), marker='o', linestyle='--') - plt.xlabel('Number of Components') - plt.ylabel('Cumulative Explained Variance') - plt.title('Explained_Variance_By_Number_Of_Principal_Components') - plt.grid(True) - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', 'Explained_Variance.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"Explained variance plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', 'Explained_Variance.png')}") - - # 3D PCA plot - logging.info("Creating 3D PCA plot") - from mpl_toolkits.mplot3d import Axes3D - fig = plt.figure(figsize=(12, 8)) - ax = fig.add_subplot(111, projection='3d') - pca_coords_3d = pca.transform(vectorizer.transform(self.texts.values()).toarray())[:, :3] - ax.scatter(pca_coords_3d[:, 0], pca_coords_3d[:, 1], pca_coords_3d[:, 2], c='b', marker='o') - for i, file in enumerate(self.texts.keys()): - ax.text(pca_coords_3d[i, 0], pca_coords_3d[i, 1], pca_coords_3d[i, 2], file, fontsize=8) - ax.set_xlabel('PC1') - ax.set_ylabel('PC2') - ax.set_zlabel('PC3') - ax.set_title('3D_PCA_Plot') - plt.tight_layout() - plt.savefig(os.path.join(self.output_dir, 'PCA_Analysis', '3D_PCA_Plot.png'), dpi=300, bbox_inches='tight') - plt.close() - logging.info(f"3D PCA plot saved: {os.path.join(self.output_dir, 'PCA_Analysis', '3D_PCA_Plot.png')}") def entity_domain_analysis(self): + if self._outputs_exist(('Heatmaps/Entity_Domain_Combination_Heatmap.png',)): + return logging.info("Analyzing entity-domain combinations") entities = set() from_domains = set() @@ -419,6 +469,8 @@ def entity_domain_analysis(self): def generate_word_clouds(self): logging.info("Generating word clouds") for file, text in self.texts.items(): + if self._outputs_exist((f'WordClouds/{file}_wordcloud.png',)): + continue wordcloud = WordCloud(stopwords=self.stop_words, background_color='white', width=800, height=400).generate(text) plt.figure(figsize=(10, 5)) plt.imshow(wordcloud, interpolation='bilinear') @@ -430,6 +482,8 @@ def generate_word_clouds(self): logging.info(f"Word cloud saved: {os.path.join(self.output_dir, 'WordClouds', f'{file}_wordcloud.png')}") def plot_top_terms_bar_chart(self, top_terms, num_terms=30): + if self._outputs_exist((f'TopTerms/Top_{num_terms}_Terms_Bar_Chart.png',)): + return logging.info(f"Plotting top {num_terms} terms bar chart") # Convert Series to list and flatten @@ -460,6 +514,8 @@ def plot_top_terms_bar_chart(self, top_terms, num_terms=30): logging.info(f"Top {num_terms} terms bar chart saved: {os.path.join(self.output_dir, 'TopTerms', f'Top_{num_terms}_Terms_Bar_Chart.png')}") def plot_term_frequency_heatmap(self, tfidf_matrix, feature_names): + if self._outputs_exist(('Heatmaps/Term_Frequency_Heatmap.png',)): + return logging.info("Plotting term frequency heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) plt.figure(figsize=(20, 10)) @@ -473,6 +529,8 @@ def plot_term_frequency_heatmap(self, tfidf_matrix, feature_names): logging.info(f"Term frequency heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', 'Term_Frequency_Heatmap.png')}") def calculate_diversity_indices(self, tfidf_matrix): + if self._outputs_exist(('Diversity/Shannon_Diversity_Indices.png',)): + return logging.info("Calculating diversity indices") df = pd.DataFrame(tfidf_matrix.toarray(), index=self.texts.keys()) @@ -502,6 +560,8 @@ def calculate_diversity_indices(self, tfidf_matrix): logging.info(f"Diversity indices plot saved: {os.path.join(self.output_dir, 'Diversity', 'Shannon_Diversity_Indices.png')}") def plot_term_network(self, tfidf_matrix, feature_names): + if self._outputs_exist(('Networks/Term_Network_Graph.png',)): + return logging.info("Plotting term network graph") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) G = nx.Graph() @@ -527,6 +587,8 @@ def plot_term_network(self, tfidf_matrix, feature_names): logging.info(f"Term network graph saved: {os.path.join(self.output_dir, 'Networks', 'Term_Network_Graph.png')}") def plot_term_correlation_heatmap(self, tfidf_matrix, feature_names): + if self._outputs_exist(('Heatmaps/Term_Correlation_Heatmap.png',)): + return logging.info("Plotting term correlation heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) @@ -543,6 +605,8 @@ def plot_term_correlation_heatmap(self, tfidf_matrix, feature_names): logging.info(f"Term correlation heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', 'Term_Correlation_Heatmap.png')}") def plot_document_similarity_heatmap(self, tfidf_matrix): + if self._outputs_exist(('Heatmaps/Document_Similarity_Heatmap.png',)): + return logging.info("Plotting document similarity heatmap") similarity_matrix = (tfidf_matrix * tfidf_matrix.T).toarray() @@ -556,6 +620,8 @@ def plot_document_similarity_heatmap(self, tfidf_matrix): logging.info(f"Document similarity heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', 'Document_Similarity_Heatmap.png')}") def plot_term_document_heatmap(self, tfidf_matrix, feature_names, top_n=50): + if self._outputs_exist((f'Heatmaps/Top_{top_n}_Terms_Document_Heatmap.png',)): + return logging.info(f"Plotting top {top_n} terms-document heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) @@ -576,6 +642,8 @@ def plot_term_document_heatmap(self, tfidf_matrix, feature_names, top_n=50): logging.info(f"Top {top_n} terms-document heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', f'Top_{top_n}_Terms_Document_Heatmap.png')}") def plot_term_cooccurrence_heatmap(self, tfidf_matrix, feature_names, top_n=50): + if self._outputs_exist((f'Heatmaps/Top_{top_n}_Terms_Cooccurrence_Heatmap.png',)): + return logging.info(f"Plotting top {top_n} terms co-occurrence heatmap") df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names, index=self.texts.keys()) @@ -598,43 +666,32 @@ def plot_term_cooccurrence_heatmap(self, tfidf_matrix, feature_names, top_n=50): logging.info(f"Top {top_n} terms co-occurrence heatmap saved: {os.path.join(self.output_dir, 'Heatmaps', f'Top_{top_n}_Terms_Cooccurrence_Heatmap.png')}") def run_analysis(self): - print("Performing TF-IDF analysis...") - top_terms, tfidf_matrix, vectorizer = self.perform_tfidf_analysis() - print("Top terms for each document:") - print(top_terms) - - print("Plotting top terms bar chart...") - self.plot_top_terms_bar_chart(top_terms, num_terms=30) - - print("Plotting term frequency heatmap...") - self.plot_term_frequency_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Calculating diversity indices...") - self.calculate_diversity_indices(tfidf_matrix) - - print("Clustering documents...") - self.cluster_documents() - - print("Analyzing entity-domain combinations...") - self.entity_domain_analysis() - - print("Generating word clouds...") - self.generate_word_clouds() - - print("Plotting term network graph...") - self.plot_term_network(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Plotting term correlation heatmap...") - self.plot_term_correlation_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Plotting document similarity heatmap...") - self.plot_document_similarity_heatmap(tfidf_matrix) - - print("Plotting top terms-document heatmap...") - self.plot_term_document_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) - - print("Plotting top terms co-occurrence heatmap...") - self.plot_term_cooccurrence_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if self._outputs_exist(self._analysis_outputs()): + return + if not self._outputs_exist(self._tfidf_outputs()): + top_terms, tfidf_matrix, vectorizer = self.perform_tfidf_analysis() + if not self._outputs_exist(('TopTerms/Top_30_Terms_Bar_Chart.png',)): + self.plot_top_terms_bar_chart(top_terms, num_terms=30) + if not self._outputs_exist(('Heatmaps/Term_Frequency_Heatmap.png',)): + self.plot_term_frequency_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Diversity/Shannon_Diversity_Indices.png',)): + self.calculate_diversity_indices(tfidf_matrix) + if not self._outputs_exist(('Networks/Term_Network_Graph.png',)): + self.plot_term_network(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Heatmaps/Term_Correlation_Heatmap.png',)): + self.plot_term_correlation_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Heatmaps/Document_Similarity_Heatmap.png',)): + self.plot_document_similarity_heatmap(tfidf_matrix) + if not self._outputs_exist(('Heatmaps/Top_50_Terms_Document_Heatmap.png',)): + self.plot_term_document_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(('Heatmaps/Top_50_Terms_Cooccurrence_Heatmap.png',)): + self.plot_term_cooccurrence_heatmap(tfidf_matrix, vectorizer.get_feature_names_out()) + if not self._outputs_exist(self._cluster_outputs()): + self.cluster_documents() + if not self._outputs_exist(('Heatmaps/Entity_Domain_Combination_Heatmap.png',)): + self.entity_domain_analysis() + if not self._outputs_exist(tuple(f'WordClouds/{file}_wordcloud.png' for file in self.files)): + self.generate_word_clouds() def confidence_ellipse(x, y, ax, n_std=3.0, facecolor='none', **kwargs): """ diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/README.md b/1_PREPARE/Methods/Research/FieldSHIFT-2/README.md index 76b268e1..e7bb0b81 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/README.md +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/README.md @@ -13,7 +13,7 @@ Domain-shifting dissertation framework for cross-disciplinary Active Inference a | 5. Improve | `dissertation_improver.py` | Iterative quality improvement | | 6. Translate | `dissertation_translation.py` | 24-language translation | | 7. Evaluate | `dissertation_entity_evaluation.py`, `dissertation_grant_evaluator.py` | Entity and grant evaluation | -| 8. Meta-analyze | `Meta_Analysis_Shifted_Dissertation.py` | Cross-dissertation meta-analysis | +| 8. Meta-analyze | `Meta_Analysis_Shifted_Dissertation.py`, `Meta_Analysis_Shifted_Dissertation_Outline.py` | Dissertation and outline meta-analysis | ## Data Structure @@ -27,6 +27,19 @@ Domain-shifting dissertation framework for cross-disciplinary Active Inference a Arabic, Bengali, Chinese-Mandarin, English, French, German, Hebrew, Hindi, Japanese, Javanese, Korean, Navajo, Portuguese, Punjabi, Russian, Sami, Sanskrit, Spanish, Telugu, Vietnamese, and more. +## Existing-output preservation + +Artifact-writing callables preserve an existing destination before +artifact-specific input/provider/computation work, including empty existing +files. Complete batches skip generation; partial batches retain existing +artifacts and produce eligible missing companions. Both dissertation and +outline meta-analysis modules are covered. + +This is not credential-free/import-free script execution or scientific-runtime +qualification. Normal imports/entrypoints retain existing optional-library, +configuration/key-file and NLTK initialization requirements. Use only an +explicitly authorized research runtime for generation. + ## References See `FieldSHIFT-2.md` for the complete framework documentation. diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/SPEC.md b/1_PREPARE/Methods/Research/FieldSHIFT-2/SPEC.md index d614c01e..6d65a869 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/SPEC.md +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/SPEC.md @@ -30,6 +30,19 @@ Domain-shifting dissertation framework that translates research across disciplin - 📄 `shift_domains.py` - 📄 `utils.py` +## Writer contract + +The shared text writer is `save_file(content, full_path)`: two arguments, +caller-owned serialization and verbatim UTF-8 text, with no implicit +directory/filename overload. Structured linguistic results are explicitly +JSON-serialized before saving. Existing outputs, including empty files, are +preserved rather than resumed, repaired or silently overwritten. + +Multi-artifact methods guard complete groups and individual companions. +Existing PCA/projection/JSON companions do not suppress eligible missing +outputs. Scientific parameters, feature definitions, input filters, filename +conventions and historical tuple serialization are unchanged. + ## Version - **Framework**: Active InferAnts diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_entity_evaluation.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_entity_evaluation.py index 323476cf..b2c2fa28 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_entity_evaluation.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_entity_evaluation.py @@ -133,24 +133,23 @@ def process_dissertation_evaluation(input_dir: str, output_base_dir: str, llm_pa llm_params (Dict[str, Any]): LLM parameters from config. """ try: - # Load evaluator modules as plain text - evaluator_modules = [] + # Discover paths; read descriptions only for missing outputs. + evaluator_paths = [] entity_dir = os.path.join(os.path.dirname(__file__), '../Fonds/Entity') for root, dirs, files in os.walk(entity_dir): for file in files: if file.endswith('.py'): module_path = os.path.join(root, file) - with open(module_path, 'r') as f: - module_content = f.read() evaluator_name = os.path.splitext(file)[0] - evaluator_modules.append((evaluator_name, module_content)) - logger.info(f"Loaded evaluator module: {file}") + evaluator_paths.append((evaluator_name, module_path)) + logger.info(f"Discovered evaluator module: {file}") + evaluator_contents = {} for input_file in os.listdir(input_dir): if input_file.endswith(".md"): input_file_path = os.path.join(input_dir, input_file) - for evaluator_name, evaluator_content in evaluator_modules: + for evaluator_name, evaluator_path in evaluator_paths: # Prepare the output directory and file name output_dir = os.path.join(output_base_dir, f"Evaluated_by_{evaluator_name}") os.makedirs(output_dir, exist_ok=True) @@ -161,12 +160,16 @@ def process_dissertation_evaluation(input_dir: str, output_base_dir: str, llm_pa if os.path.exists(output_file_path): continue + if evaluator_path not in evaluator_contents: + with open(evaluator_path, 'r') as source: + evaluator_contents[evaluator_path] = source.read() + description = evaluator_contents[evaluator_path] # Load the existing dissertation content dissertation_content = load_file(input_file_path) # Evaluate the dissertation start_time = time.time() - evaluated_dissertation = evaluate_dissertation(dissertation_content, evaluator_name, evaluator_content, llm_params) + evaluated_dissertation = evaluate_dissertation(dissertation_content, evaluator_name, description, llm_params) end_time = time.time() elapsed_time = end_time - start_time logger.info(f"Dissertation evaluated by {evaluator_name} in {elapsed_time:.2f} seconds.") diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_grant_evaluator.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_grant_evaluator.py index 543b7f72..39899a50 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_grant_evaluator.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/dissertation_grant_evaluator.py @@ -118,24 +118,23 @@ def process_dissertation_evaluation(input_dir: str, output_base_dir: str, llm_pa llm_params (Dict[str, Any]): LLM parameters from config. """ try: - # Load grant summary files as plain text - grant_summaries = [] + # Discover paths; read descriptions only for missing outputs. + grant_paths = [] grant_dir = os.path.join(os.path.dirname(__file__), '../Fonds/Summarized_Grants') for root, dirs, files in os.walk(grant_dir): for file in files: if file.endswith('.md'): grant_path = os.path.join(root, file) - with open(grant_path, 'r') as f: - grant_content = f.read() grant_name = os.path.splitext(file)[0] - grant_summaries.append((grant_name, grant_content)) - logger.info(f"Loaded grant summary: {file}") + grant_paths.append((grant_name, grant_path)) + logger.info(f"Discovered grant summary: {file}") + grant_contents = {} for input_file in os.listdir(input_dir): if input_file.endswith(".md"): input_file_path = os.path.join(input_dir, input_file) - for grant_name, grant_content in grant_summaries: + for grant_name, grant_path in grant_paths: # Prepare the output directory and file name output_dir = os.path.join(output_base_dir, f"Evaluated_for_{grant_name}") os.makedirs(output_dir, exist_ok=True) @@ -146,12 +145,16 @@ def process_dissertation_evaluation(input_dir: str, output_base_dir: str, llm_pa if os.path.exists(output_file_path): continue + if grant_path not in grant_contents: + with open(grant_path, 'r') as source: + grant_contents[grant_path] = source.read() + description = grant_contents[grant_path] # Load the existing dissertation content dissertation_content = load_file(input_file_path) # Evaluate the dissertation start_time = time.time() - evaluated_dissertation = evaluate_dissertation(dissertation_content, grant_content, llm_params) + evaluated_dissertation = evaluate_dissertation(dissertation_content, description, llm_params) end_time = time.time() elapsed_time = end_time - start_time logger.info(f"Dissertation evaluated for {grant_name} in {elapsed_time:.2f} seconds.") diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/generate_pro_shift_domains.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/generate_pro_shift_domains.py index 53e70f15..c3fafe80 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/generate_pro_shift_domains.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/generate_pro_shift_domains.py @@ -51,6 +51,11 @@ def generate_concatenated_domains(domains: List[str], prompt: str, output_dir: s for domain_a, domain_b in product(domains, repeat=2): domain_a_name = os.path.splitext(os.path.basename(domain_a))[0] domain_b_name = os.path.splitext(os.path.basename(domain_b))[0] + output_filename = f'concatenated_{domain_a_name}_to_{domain_b_name}.md' + output_path = os.path.join(output_dir, output_filename) + if os.path.exists(output_path): + logger.info(f"Output already exists; preserving {output_path}") + continue logger.info(f"Processing combination: {domain_a_name} and {domain_b_name}") domain_a_content = load_file(domain_a) @@ -58,8 +63,6 @@ def generate_concatenated_domains(domains: List[str], prompt: str, output_dir: s concatenated_content = generate_pro_shift_domains(domain_a_content, domain_b_content, prompt) - output_filename = f'concatenated_{domain_a_name}_to_{domain_b_name}.md' - output_path = os.path.join(output_dir, output_filename) save_file(concatenated_content, output_path) logger.info(f"Concatenated domain saved as: {output_filename}") @@ -90,21 +93,25 @@ def process_domains(config: dict = DEFAULT_CONFIG) -> None: # Load domain contents domain_files = [os.path.join(input_domain_dir, f) for f in os.listdir(input_domain_dir) if f.endswith('.md')] - domains = [load_file(file) for file in domain_files] logger.info("Domains loaded successfully") logger.info("Starting concatenation process") - # Load the prompt - prompt_path = os.path.join(base_dir, 'Prompts', 'FieldSHIFT-2-Domain_Shift.md') - prompt = load_file(prompt_path) - - # Generate concatenated domains - generate_concatenated_domains(domain_files, prompt, config['output_dirs']['Concatenated_Domain']) + output_dir = config['output_dirs']['Concatenated_Domain'] + if any( + not os.path.exists(os.path.join( + output_dir, + f"concatenated_{os.path.splitext(os.path.basename(a))[0]}_to_{os.path.splitext(os.path.basename(b))[0]}.md", + )) + for a, b in product(domain_files, repeat=2) + ): + prompt_path = os.path.join(base_dir, 'Prompts', 'FieldSHIFT-2-Domain_Shift.md') + prompt = load_file(prompt_path) + generate_concatenated_domains(domain_files, prompt, output_dir) logger.info("Concatenation process completed successfully.") print(f"\n{'='*50}\nConcatenation process summary:\n{'-'*50}") - print(f"Total domains processed: {len(domains)}") + print(f"Total domains processed: {len(domain_files)}") print(f"Output directory: {config['output_dirs']['Concatenated_Domain']}") print(f"{'='*50}\n") except Exception as e: diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/shift_domains.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/shift_domains.py index 75fc90b9..d6c29eef 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/shift_domains.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/shift_domains.py @@ -37,6 +37,11 @@ def perform_domain_shift(input_file: str, output_dir: str, llm_params: dict) -> llm_params (dict): Parameters for the language model. """ try: + output_file_name = os.path.basename(input_file).replace('concatenated_', 'shifted_') + output_file_path = os.path.join(output_dir, output_file_name) + if os.path.exists(output_file_path): + logger.info(f"Output already exists; preserving {output_file_path}") + return logger.info(f"Processing file: {input_file}") # Load domain content @@ -48,9 +53,6 @@ def perform_domain_shift(input_file: str, output_dir: str, llm_params: dict) -> # Generate the shifted domain using LLM shifted_domain, _ = generate_llm_response(domain_content, llm_params) - # Prepare the output file path - output_file_name = os.path.basename(input_file).replace('concatenated_', 'shifted_') - output_file_path = os.path.join(output_dir, output_file_name) # Save the shifted domain save_file(shifted_domain, output_file_path) @@ -71,13 +73,7 @@ def process_domain_shift(input_dir: str, output_dir: str, config: dict) -> None: try: logger.info("Starting domain shift processing") - # Get the API key using the new get_api_key() function - api_key = get_api_key() - if not api_key: - raise ValueError("API key not found in .env or LLM_keys.key file") - - # Prepare LLM parameters - llm_params = {**USER_CONFIG['llm_params'], 'api_key': api_key} + llm_params = None # Iterate over all files in the input directory for input_file in os.listdir(input_dir): @@ -93,6 +89,11 @@ def process_domain_shift(input_dir: str, output_dir: str, config: dict) -> None: print(f"Output file already exists: {output_file_name}. Skipping LLM request.") else: # Perform the domain shift + if llm_params is None: + api_key = get_api_key() + if not api_key: + raise ValueError("API key not found in .env or LLM_keys.key file") + llm_params = {**USER_CONFIG['llm_params'], 'api_key': api_key} start_time = time.time() perform_domain_shift(input_file_path, output_dir, llm_params) elapsed_time = time.time() - start_time diff --git a/1_PREPARE/Methods/Research/FieldSHIFT-2/utils.py b/1_PREPARE/Methods/Research/FieldSHIFT-2/utils.py index 507fbdbc..ce57a3b4 100644 --- a/1_PREPARE/Methods/Research/FieldSHIFT-2/utils.py +++ b/1_PREPARE/Methods/Research/FieldSHIFT-2/utils.py @@ -43,11 +43,16 @@ def save_file(content: str, file_path: str) -> None: Raises: IOError: If there's an error writing to the file. """ + if os.path.exists(file_path): + logging.info(f"Output already exists; preserving {file_path}") + return try: - with open(file_path, 'w', encoding='utf-8') as file: + with open(file_path, 'x', encoding='utf-8') as file: file.write(content) logging.info(f"File saved successfully: {file_path}") + except FileExistsError: + logging.info(f"Output already exists; preserving {file_path}") except IOError as e: logging.error(f"Error writing to file {file_path}: {str(e)}") raise @@ -196,6 +201,9 @@ def process_pro_shifted_domains(input_dir: str, output_dir: str, llm_params: Dic input_path = os.path.join(input_dir, filename) output_filename = f"shifted_{filename}" output_path = os.path.join(output_dir, output_filename) + if os.path.exists(output_path): + logger.info(f"Output already exists; preserving {output_path}") + continue logger.info(f"Processing file: {filename}") @@ -207,8 +215,7 @@ def process_pro_shifted_domains(input_dir: str, output_dir: str, llm_params: Dic shifted_content, time_taken = generate_llm_response(prompt, llm_params) - with open(output_path, 'w', encoding='utf-8') as file: - file.write(shifted_content) + save_file(shifted_content, output_path) logger.info(f"Processed {filename} in {time_taken:.2f} seconds. Output saved as {output_filename}") @@ -315,7 +322,15 @@ def generate_entity_library() -> Dict[str, Any]: def save_entity_library(library: Dict[str, Any], filename: str): """Save the entity library to a JSON file.""" - os.makedirs(os.path.dirname(filename), exist_ok=True) - with open(filename, 'w') as f: - json.dump(library, f, indent=2) - logger.info(f"Entity library saved to {filename}") \ No newline at end of file + if os.path.exists(filename): + logging.info(f"Output already exists; preserving {filename}") + return + parent = os.path.dirname(filename) + if parent: + os.makedirs(parent, exist_ok=True) + try: + with open(filename, 'x') as f: + json.dump(library, f, indent=2) + except FileExistsError: + return + logging.info(f"Entity library saved to {filename}") \ No newline at end of file diff --git a/2_OPERATE/MetaInformAnt_Simulation.py b/2_OPERATE/MetaInformAnt_Simulation.py index 9045b399..8cdc9090 100644 --- a/2_OPERATE/MetaInformAnt_Simulation.py +++ b/2_OPERATE/MetaInformAnt_Simulation.py @@ -19,19 +19,12 @@ from typing import Any import numpy as np -from environment import Environment +from environment import GRID_MOVES, Environment from active_infer_ants import ActiveInferenceAgent logger = logging.getLogger(__name__) -_MOVEMENT = [ - (0, 0), # stay - (1, 0), # east - (-1, 0), # west - (0, 1), # south - (0, -1), # north -] class MetaInformAntSimulation: @@ -92,7 +85,7 @@ def __init__( self.colony = None self.agents = [ ActiveInferenceAgent( - n_dims=len(_MOVEMENT), + n_dims=len(GRID_MOVES), seed=int(self.rng.integers(0, 2**31 - 1)), **self.agent_params, ) @@ -163,8 +156,8 @@ def _act( ) -> float: """Move an agent according to its action vector and deposit pheromone.""" move_index = int(np.argmax(np.asarray(action))) - move_index = min(max(move_index, 0), len(_MOVEMENT) - 1) - dx, dy = _MOVEMENT[move_index] + move_index = min(max(move_index, 0), len(GRID_MOVES) - 1) + dx, dy = GRID_MOVES[move_index] nx = int(np.clip(x + dx, 0, self.environment.width - 1)) ny = int(np.clip(y + dy, 0, self.environment.height - 1)) self.positions[index] = (nx, ny) diff --git a/2_OPERATE/environment.py b/2_OPERATE/environment.py index 7fcbc028..080669c6 100644 --- a/2_OPERATE/environment.py +++ b/2_OPERATE/environment.py @@ -18,6 +18,9 @@ logger = logging.getLogger(__name__) +# Observation and action coordinates: stay, east, west, south, north. +GRID_MOVES = ((0, 0), (1, 0), (-1, 0), (0, 1), (0, -1)) + class Environment: """A 2D grid world for ant-colony active-inference simulations. @@ -98,19 +101,20 @@ def sense_pheromone(self, x: int, y: int) -> np.ndarray: """Return the local pheromone signal around a cell as a vector. The signal is the 5-cell cross centred on the agent's position - (centre, north, south, east, west), which matches the 5-move action - space used by :class:`MetaInformAntSimulation`. + (centre, east, west, south, north), in ``GRID_MOVES`` action order. """ cx = int(np.clip(x, 0, self.width - 1)) cy = int(np.clip(y, 0, self.height - 1)) - cross = [ - (cy, cx), - (max(0, cy - 1), cx), - (min(self.height - 1, cy + 1), cx), - (cy, min(self.width - 1, cx + 1)), - (cy, max(0, cx - 1)), - ] - return np.asarray([self.pheromone[yy, xx] for yy, xx in cross], dtype=float) + return np.asarray( + [ + self.pheromone[ + min(max(cy + dy, 0), self.height - 1), + min(max(cx + dx, 0), self.width - 1), + ] + for dx, dy in GRID_MOVES + ], + dtype=float, + ) def nearest_food_distance(self, x: int, y: int) -> float: """Euclidean distance to the nearest food source.""" diff --git a/2_OPERATE/execute_Simulation.py b/2_OPERATE/execute_Simulation.py index 19c13ccc..838caeda 100644 --- a/2_OPERATE/execute_Simulation.py +++ b/2_OPERATE/execute_Simulation.py @@ -170,8 +170,6 @@ def conclude_simulation(self) -> None: ), "Simulation conclusion", ) - self.data_logger.finalize() - self.performance_monitor.stop() self._generate_final_report() def _generate_final_report(self) -> None: @@ -200,6 +198,7 @@ def _save_report(self, report: str) -> None: self.logger.info("Simulation report saved to %s", report_file) except OSError as e: self.logger.error("Failed to save simulation report: %s", e) + raise def run(self) -> None: """Run the complete simulation sequence.""" @@ -211,9 +210,37 @@ def run(self) -> None: ("Simulation conclusion", self.conclude_simulation), ] - for description, step_function in simulation_steps: - self.logger.info("%s in progress.", description) - step_function() + try: + for description, step_function in simulation_steps: + self.logger.info("%s in progress.", description) + step_function() + except BaseException: + try: + self._cleanup() + except Exception: + self.logger.exception("Cleanup failed while preserving the original failure") + raise + else: + self._cleanup() + + def _cleanup(self) -> None: + """Attempt every owned cleanup; do not turn cleanup errors into success.""" + first_error = None + operations = [ + self.performance_monitor.stop, + self.data_logger.finalize, + ] + if self.renderer is not None: + operations.append(self.renderer.close) + for operation in operations: + try: + operation() + except Exception as exc: + self.logger.exception("Simulation cleanup failed") + if first_error is None: + first_error = exc + if first_error is not None: + raise first_error def _safely_execute(self, operation: Callable[[], None], description: str) -> None: """Execute a callable operation safely, handling exceptions.""" @@ -230,14 +257,10 @@ def main() -> None: executor.run() except SimulationError as se: logging.critical("Simulation failed: %s", se, exc_info=True) + raise except Exception as e: logging.critical("Unexpected error: %s", e, exc_info=True) - finally: - # Perform necessary cleanup. - if executor.performance_monitor: - executor.performance_monitor.stop() - if executor.data_logger: - executor.data_logger.finalize() + raise if __name__ == "__main__": diff --git a/2_OPERATE/plan_Simulation.py b/2_OPERATE/plan_Simulation.py index 47f655cd..09397b6a 100644 --- a/2_OPERATE/plan_Simulation.py +++ b/2_OPERATE/plan_Simulation.py @@ -12,9 +12,11 @@ from __future__ import annotations +import json import logging import os import sys +from copy import deepcopy from typing import Any import numpy as np @@ -66,6 +68,35 @@ def __init__(self, config_path: str | None = None, seed: int | None = None) -> N self.config_path = config_path self.seed = _DEFAULT_SEED if seed is None else int(seed) self.rng = np.random.default_rng(self.seed) + if config_path is None: + selected = deepcopy( + { + "SIMULATION_SETTINGS": config.SIMULATION_SETTINGS, + "ENVIRONMENT_CONFIG": config.ENVIRONMENT_CONFIG, + "ANT_AND_COLONY_CONFIG": config.ANT_AND_COLONY_CONFIG, + "META_CONFIG": metaconfig.META_CONFIG, + } + ) + else: + with open(config_path, encoding="utf-8") as stream: + selected = json.load(stream) + if not isinstance(selected, dict): + raise ValueError("Simulation configuration must be an object") + for key in ("SIMULATION_SETTINGS", "ENVIRONMENT_CONFIG"): + if key not in selected: + raise ValueError(f"Missing simulation configuration section: {key}") + for key in ( + "SIMULATION_SETTINGS", + "ENVIRONMENT_CONFIG", + "ANT_AND_COLONY_CONFIG", + "META_CONFIG", + ): + if not isinstance(selected.get(key, {}), dict): + raise ValueError(f"Simulation configuration section must be an object: {key}") + self.settings = selected["SIMULATION_SETTINGS"] + self.environment_config = selected["ENVIRONMENT_CONFIG"] + self.ant_and_colony_config = selected.get("ANT_AND_COLONY_CONFIG", {}) + self.meta_config = selected.get("META_CONFIG", {}) self.simulation_environment: Environment | None = None self.agent_params: dict[str, Any] = {} @@ -87,11 +118,8 @@ def __init__(self, config_path: str | None = None, seed: int | None = None) -> N # ------------------------------------------------------------------ # def initialize_components(self) -> None: """Initialize all planning components from configuration.""" - settings = config.SIMULATION_SETTINGS - environment_config = config.ENVIRONMENT_CONFIG - self.simulation_environment = build_environment_from_config( - settings, environment_config, seed=self.seed - ) + settings = self.settings + self.simulation_environment = None self.agent_params = self.get_agent_params() self.niche_params = self.get_niche_params() self.parallel_execution = self.get_parallel_execution_settings() @@ -111,11 +139,11 @@ def get_agent_params(self) -> dict[str, Any]: fall back to the consolidated ant/colony config so the planner never assumes a key that is not there. """ - ant_colony = metaconfig.META_CONFIG.get("ANT_AND_COLONY", {}) + ant_colony = self.meta_config.get("ANT_AND_COLONY", {}) params = ant_colony.get("AGENT_PARAMS") if isinstance(params, dict): return dict(params) - nestmate = config.ANT_AND_COLONY_CONFIG.get("NESTMATE", {}) + nestmate = self.ant_and_colony_config.get("NESTMATE", {}) active_inference = nestmate.get("ACTIVE_INFERENCE", {}) # Only pass parameters the agent API actually accepts. known = ("sensory_precision", "prior_precision", "learning_rate") @@ -123,16 +151,16 @@ def get_agent_params(self) -> dict[str, Any]: def get_niche_params(self) -> dict[str, Any]: """Return niche parameters from the meta configuration (if present).""" - ant_colony = metaconfig.META_CONFIG.get("ANT_AND_COLONY", {}) + ant_colony = self.meta_config.get("ANT_AND_COLONY", {}) params = ant_colony.get("NICHE_PARAMS") return dict(params) if isinstance(params, dict) else {} def get_parallel_execution_settings(self) -> dict[str, Any]: """Return validated parallel-execution settings from the config.""" - settings = dict(config.SIMULATION_SETTINGS.get("PARALLEL_EXECUTION", {})) + settings = dict(self.settings.get("PARALLEL_EXECUTION", {})) enabled = bool(settings.get("ENABLED", False)) worker_count = int(settings.get("WORKER_COUNT", 1) or 1) - agent_count = int(config.SIMULATION_SETTINGS.get("AGENT_COUNT", 1)) + agent_count = int(self.settings.get("AGENT_COUNT", 1)) if enabled and worker_count > agent_count: logger.warning( "Worker count (%d) exceeds agent count (%d); clamping", @@ -144,10 +172,10 @@ def get_parallel_execution_settings(self) -> dict[str, Any]: def estimate_computational_load(self) -> dict[str, Any]: """Estimate the computational cost of the planned run.""" - settings = config.SIMULATION_SETTINGS + settings = self.settings return estimate_computational_resources( num_agents=int(settings.get("AGENT_COUNT", 100)), - num_food_sources=len(config.ENVIRONMENT_CONFIG.get("RESOURCE_ZONES", [])), + num_food_sources=len(self.environment_config.get("RESOURCE_ZONES", [])), num_nests=int(settings.get("NEST_COUNT", 1)), max_steps=int(settings.get("MAX_STEPS", 500)), parallel_execution=self.parallel_execution, @@ -160,17 +188,17 @@ def create_simulation( self, max_steps: int | None = None ) -> MetaInformAntSimulation: """Create and return a configured ``MetaInformAntSimulation``.""" - if self.simulation_environment is None: - self.initialize_components() - assert self.simulation_environment is not None + self.simulation_environment = build_environment_from_config( + self.settings, self.environment_config, seed=self.seed + ) if max_steps is not None: self.simulation_environment.max_steps = int(max_steps) - settings = config.SIMULATION_SETTINGS + settings = self.settings simulation = MetaInformAntSimulation( num_agents=int(settings.get("AGENT_COUNT", 100)), simulation_environment=self.simulation_environment, - num_food_sources=len(config.ENVIRONMENT_CONFIG.get("RESOURCE_ZONES", [])), + num_food_sources=len(self.environment_config.get("RESOURCE_ZONES", [])), num_nests=int(settings.get("NEST_COUNT", 1)), agent_params=self.agent_params, niche_params=self.niche_params, @@ -240,7 +268,7 @@ def run_multiple_simulations(self, num_simulations: int) -> list[dict[str, Any]] if __name__ == "__main__": setup = SimulationPlanner() - num_simulations = int(config.SIMULATION_SETTINGS.get("NUM_SIMULATIONS", 1) or 1) + num_simulations = int(setup.settings.get("NUM_SIMULATIONS", 1) or 1) if num_simulations > 1: results_all = setup.run_multiple_simulations(num_simulations) logger.info("Completed %d simulations.", num_simulations) diff --git a/2_OPERATE/render_Simulation.py b/2_OPERATE/render_Simulation.py index 0e5022fb..7813a653 100644 --- a/2_OPERATE/render_Simulation.py +++ b/2_OPERATE/render_Simulation.py @@ -28,6 +28,8 @@ def __init__( self.agents = agents self.fig, self.ax = plt.subplots(figsize=(14, 12)) self.colormap = self._create_custom_colormap() + self._colorbar = None + self._convergence_ax = None self._configure_plot_environment() self.animation: Optional[animation.FuncAnimation] = None logging.info("SimulationRenderer initialized successfully.") @@ -75,28 +77,13 @@ def _configure_plot_environment(self) -> None: def _plot_entities( self, - entities: List[Entity], + positions: List[Tuple[float, float]], color: str, label: str, marker: str = "o", size: int = 50, ) -> None: - """Plot entities on the simulation environment. - - Parameters - ---------- - entities : List[Entity] - The list of entities to plot. - color : str - Color of the entities. - label : str - Label for the entities. - marker : str, optional - Marker style, by default 'o'. - size : int, optional - Size of the markers, by default 50. - """ - positions = [(entity.x, entity.y) for entity in entities] + """Plot coordinates without snapshotting the live agent projection.""" if positions: x, y = zip(*positions) self.ax.scatter( @@ -110,7 +97,6 @@ def _plot_entities( s=size, zorder=3, ) - logging.debug(f"Plotted {label} with color {color}.") def _plot_pheromone_trails(self) -> None: """Plot pheromone trails as a heatmap with enhanced visualization.""" @@ -128,14 +114,20 @@ def _plot_pheromone_trails(self) -> None: interpolation="gaussian", zorder=1, ) - colorbar = self.fig.colorbar( - heatmap, ax=self.ax, label="Pheromone Intensity", pad=0.02 - ) - colorbar.ax.tick_params(labelsize=12) - colorbar.set_label("Pheromone Intensity", fontsize=14, fontweight="bold") + if self._colorbar is None: + self._colorbar = self.fig.colorbar( + heatmap, ax=self.ax, label="Pheromone Intensity", pad=0.16 + ) + self._colorbar.ax.tick_params(labelsize=12) + self._colorbar.set_label("Pheromone Intensity", fontsize=14, fontweight="bold") + else: + self._colorbar.update_normal(heatmap) def _refresh_environment(self, frame: Optional[int] = None) -> List[plt.Artist]: """Refresh the environment for the next frame or step.""" + if self._convergence_ax is not None: + self._convergence_ax.remove() + self._convergence_ax = None self.ax.clear() self._configure_plot_environment() self._plot_pheromone_trails() @@ -147,8 +139,14 @@ def _refresh_environment(self, frame: Optional[int] = None) -> List[plt.Artist]: fontweight="bold", ) - self._plot_entities(self.nests, "red", "Nests", marker="s", size=100) - self._plot_entities(self.agents, "black", "Ants", marker="^", size=80) + self._plot_entities( + [(nest.x, nest.y) for nest in self.nests], + "red", "Nests", marker="s", size=100, + ) + positions = getattr(self.simulation_environment, "agent_positions", None) + if positions is None: + positions = [(agent.x, agent.y) for agent in self.agents] + self._plot_entities(positions, "black", "Ants", marker="^", size=80) legend = self.ax.legend( loc="upper right", fontsize=14, framealpha=0.9, edgecolor="black" @@ -205,6 +203,7 @@ def render_post_simulation(self, simulation_results: Dict[str, Any]) -> None: if "convergence" in simulation_results: convergence_data = simulation_results["convergence"] ax2 = self.ax.twinx() + self._convergence_ax = ax2 ax2.plot(convergence_data, "r--", label="Convergence", linewidth=2) ax2.set_ylabel( "Convergence Metric", color="r", fontsize=16, fontweight="bold" @@ -240,15 +239,24 @@ def _add_performance_textbox(self, metrics: Dict[str, float]) -> None: [f"{k.replace('_', ' ').title()}: {v:.2f}" for k, v in metrics.items()] ) self.ax.text( - 0.02, + 0.98, 0.98, text_box, transform=self.ax.transAxes, fontsize=12, verticalalignment="top", + horizontalalignment="right", bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.8), ) + def close(self) -> None: + """Stop the owned animation and close only this renderer's figure.""" + if self.animation is not None: + if self.animation.event_source is not None: + self.animation.event_source.stop() + self.animation = None + plt.close(self.fig) + def save_animation(self, filename: str, fps: int = 30, dpi: int = 300) -> None: """Save the animation to a file. diff --git a/4_REPORT/AGENTS.md b/4_REPORT/AGENTS.md index 751d87b5..08ad59ad 100644 --- a/4_REPORT/AGENTS.md +++ b/4_REPORT/AGENTS.md @@ -4,18 +4,24 @@ ## Purpose -Report generation, specification, and communication of results. This phase transforms measured outcomes into structured reports. +Report specifications and asynchronous orchestration. End-to-end execution +requires real collection, analysis, generation, delivery and error providers; +those implementations are not included here. The contract requires +`DataCollector`, `DataAnalyzer`, `ReportGenerator`, `ReportDeliverer`, +reporting-specific errors and qualified dependencies/configuration. ## Directory Contents - 📄 `specify_report.py` — `DataSourceType` enum + report specification dataclasses: defines report structure, data sources, and JSON serialization with UUID tracking -- 📄 `execute_report.py` — `ReportExecutor` class: async report generation with `asyncio`, logging, and multi-format output +- `execute_report.py` — `ReportExecutor`: provider-dependent asynchronous orchestration, not a standalone reporting CLI. ## Agent Instructions - Maintain report specification schema consistency with `3_MEASURE` output types (`MeasurementMethod`, `QRFParameters`). -- Ensure `specify_report.py` `DataSourceType` enum matches actual available data sources. -- Preserve async execution patterns in `ReportExecutor`. +- Treat `DataSourceType` as specification vocabulary, not provider availability. +- Preserve asynchronous provider orchestration while consuming synchronous + `ReportSpecification.validate()`. Require real provider ownership and explicit + integration acceptance; no success substitutes or unrelated phase modules. - Track report output formats and their consumers in `5_FOLLOWUP`. - Follow the repository's CC BY-NC-ND 4.0 license. - Keep all documentation synchronized with actual contents. @@ -25,3 +31,4 @@ Report generation, specification, and communication of results. This phase trans - [ ] All files documented and up-to-date - [ ] Report schema matches 3_MEASURE outputs - [ ] README.md and SPEC.md synchronized with this AGENTS.md +- [ ] Provider-dependent execution and missing provider/dependency requirements are explicit. diff --git a/4_REPORT/README.md b/4_REPORT/README.md index e01acb8a..991edb74 100644 --- a/4_REPORT/README.md +++ b/4_REPORT/README.md @@ -1,13 +1,21 @@ # Reporting & Communication -Report generation, specification, and communication of results. This phase transforms measured outcomes into structured reports. +Report specifications and asynchronous reporting orchestration. End-to-end +execution requires real collection, analysis, generation, delivery and error +providers; those implementations are not included here. ## Modules | Module | Description | |---|---| | `specify_report.py` | `DataSourceType` enum + report specification dataclasses — defines report structure, sources, and JSON serialization with UUID tracking | -| `execute_report.py` | `ReportExecutor` class — async report generation with `asyncio`, logging, and multi-format output | +| `execute_report.py` | `ReportExecutor`: provider-dependent asynchronous orchestration, not a standalone reporting CLI | + +The reporting contract requires `DataCollector`, `DataAnalyzer`, +`ReportGenerator`, `ReportDeliverer`, reporting-specific error types, and +qualified dependencies/configuration. Specification enums and synchronous +validation do not establish provider availability. Do not add success stubs +or substitute unrelated phase modules. ## Pipeline Position @@ -15,8 +23,8 @@ Report generation, specification, and communication of results. This phase trans 3_MEASURE → [4_REPORT] → 5_FOLLOWUP ``` -- **Input**: Statistical summaries, categorized results, and metrics from `3_MEASURE` -- **Output**: Formatted reports for stakeholder communication, consumed by `5_FOLLOWUP` +- **Intended input**: Measured summaries and metrics. +- **Intended provider-dependent output**: Formatted stakeholder reports and subsequent follow-up. ## Related diff --git a/4_REPORT/execute_report.py b/4_REPORT/execute_report.py index 1090ca4f..abee9b70 100644 --- a/4_REPORT/execute_report.py +++ b/4_REPORT/execute_report.py @@ -76,7 +76,7 @@ async def execute_report(self) -> Dict[str, Any]: try: # Validate report specification - if not await self.report_specification.validate(): + if not self.report_specification.validate(): raise ValueError("Invalid report specification") # Collect data diff --git a/4_REPORT/specify_report.py b/4_REPORT/specify_report.py index 216301a1..004963ed 100644 --- a/4_REPORT/specify_report.py +++ b/4_REPORT/specify_report.py @@ -75,7 +75,6 @@ def to_dict(self) -> Dict[str, Any]: class DataSource(ReportComponent): """Detailed representation of a data source.""" - id: str = field(default_factory=lambda: str(uuid.uuid4())) name: str type: DataSourceType description: str @@ -83,6 +82,7 @@ class DataSource(ReportComponent): reliability: float # 0.0 to 1.0 update_frequency: str access_method: str + id: str = field(default_factory=lambda: str(uuid.uuid4())) last_updated: datetime = field(default_factory=datetime.now) data_schema: Optional[Dict[str, Any]] = None sample_data: Optional[Any] = None @@ -111,13 +111,13 @@ def to_dict(self) -> Dict[str, Any]: class AnalysisMethod(ReportComponent): """Detailed representation of an analysis method.""" - id: str = field(default_factory=lambda: str(uuid.uuid4())) name: str type: AnalysisMethodType description: str expected_outcomes: str limitations: str required_tools: List[str] + id: str = field(default_factory=lambda: str(uuid.uuid4())) complexity: int = field(default=1) # 1-5 scale execution_time: str = field(default="Unknown") validation_method: Optional[str] = None @@ -148,12 +148,12 @@ def to_dict(self) -> Dict[str, Any]: class TargetAudience(ReportComponent): """Detailed representation of the target audience.""" - id: str = field(default_factory=lambda: str(uuid.uuid4())) name: str expertise_level: str key_interests: List[str] preferred_formats: List[DeliveryFormatType] decision_making_context: str + id: str = field(default_factory=lambda: str(uuid.uuid4())) technical_proficiency: int = field(default=3) # 1-5 scale data_literacy: int = field(default=3) # 1-5 scale primary_language: str = field(default="English") @@ -186,11 +186,11 @@ def to_dict(self) -> Dict[str, Any]: class DeliveryFormat(ReportComponent): """Detailed representation of the delivery format.""" - id: str = field(default_factory=lambda: str(uuid.uuid4())) type: DeliveryFormatType reasoning: str alternatives: List[DeliveryFormatType] accessibility_considerations: str + id: str = field(default_factory=lambda: str(uuid.uuid4())) interactivity_level: int = field(default=1) # 1-5 scale update_frequency: Optional[str] = None version_control: bool = False diff --git a/5_FOLLOWUP/AGENTS.md b/5_FOLLOWUP/AGENTS.md index d6132fc3..4b84abe5 100644 --- a/5_FOLLOWUP/AGENTS.md +++ b/5_FOLLOWUP/AGENTS.md @@ -4,17 +4,25 @@ ## Purpose -Follow-up actions, continuous improvement planning, and iterative refinement. This phase closes the loop between reporting and future operations. +Follow-up specifications, local text/HTML formatting and provider-dependent +orchestration. Service-backed execution requires real providers not included here. ## Directory Contents -- 📄 `specify_followup.py` — Follow-up specification types: `FollowUpType`, `SessionType`, `UpdateArea` enums; `Session`, `Stakeholder`, `Resource` dataclasses; `TextReportGenerator` and `HTMLReportGenerator` classes -- 📄 `execute_followup.py` — `FollowUpExecutor` class: orchestrates full follow-up workflow integrating 7 services — `EmailService`, `ProjectManager`, `MetricsAnalyzer`, `ResourceAllocator`, `NotificationService`, `DatabaseService`, `TaskScheduler` +- `specify_followup.py` — `FollowUpType`, `SessionType`, `SessionModality` enums; + `UpdateArea`, `Session`, `Stakeholder`, `Resource` dataclasses; + `FollowUpSpecification` and local text/HTML formatters. +- `execute_followup.py` — `FollowUpExecutor`: provider-dependent orchestration + requiring `EmailService`, `ProjectManager`, `MetricsAnalyzer`, + `ResourceAllocator`, `NotificationService`, `DatabaseService`, + `TaskScheduler` and `ErrorHandler`. ## Agent Instructions -- Maintain the `FollowUpSpecification` dataclass hierarchy in `specify_followup.py`. -- Ensure `FollowUpExecutor` correctly consumes all specification types and orchestrates all 7 services. +- Maintain declared specification and typed serialization contracts. +- Verify service-backed orchestration only with real configured providers and + explicit acceptance; service names alone do not establish availability. + Distinguish local formatting/finalization from external effects. - Preserve the session lifecycle: prepare → run → post-session tasks → status update. - Track the feedback loop: follow-up outputs should inform future `1_PREPARE` configurations. - Follow the repository's CC BY-NC-ND 4.0 license. @@ -23,5 +31,5 @@ Follow-up actions, continuous improvement planning, and iterative refinement. Th ## Quality Checklist - [ ] All files documented and up-to-date -- [ ] All 7 service integrations verified +- [ ] Provider-dependent execution is explicitly distinguished from local specification/report formatting. - [ ] README.md and SPEC.md synchronized with this AGENTS.md diff --git a/5_FOLLOWUP/README.md b/5_FOLLOWUP/README.md index b799d80a..6778575c 100644 --- a/5_FOLLOWUP/README.md +++ b/5_FOLLOWUP/README.md @@ -1,13 +1,29 @@ # Continuous Improvement -Follow-up actions, continuous improvement planning, and iterative refinement. This phase closes the loop between reporting and future operations. +Follow-up specifications, local text/HTML formatting and provider-dependent +orchestration. Service-backed execution requires real providers not included here. ## Modules | Module | Description | |---|---| -| `specify_followup.py` | `FollowUpType`, `SessionType`, `UpdateArea` enums; `Session`, `Stakeholder`, `Resource` dataclasses; `TextReportGenerator` and `HTMLReportGenerator` classes | -| `execute_followup.py` | `FollowUpExecutor` class — orchestrates full follow-up workflow integrating 7 services: `EmailService`, `ProjectManager`, `MetricsAnalyzer`, `ResourceAllocator`, `NotificationService`, `DatabaseService`, `TaskScheduler` | +| `specify_followup.py` | `FollowUpType`, `SessionType`, `SessionModality`; `UpdateArea`, `Session`, `Stakeholder`, `Resource`; `FollowUpSpecification` and local text/HTML formatters | +| `execute_followup.py` | `FollowUpExecutor` requires seven service providers plus `ErrorHandler`; these are not bundled integrations | + +Execution requires `EmailService`, `ProjectManager`, `MetricsAnalyzer`, +`ResourceAllocator`, `NotificationService`, `DatabaseService`, `TaskScheduler` +and `ErrorHandler`. Provider ownership, dependencies, configuration, effects +and acceptance must be supplied explicitly. + +`SessionType` describes purpose; required `SessionModality` describes `VIRTUAL` +or `IN_PERSON` delivery. Updates use `UpdateArea.notes`. Canonical JSON stores +enum names, duration seconds, ISO timestamps and nullable dates; import restores +declared typed fields. Historical `default=str` artifacts are not automatically +rewritten or heuristically migrated. + +Handled component failures prevent successful finalization. Local finalization +records actual specification state and a local UTC timestamp; it does not invent +external persistence or notification effects. ## Pipeline Position @@ -15,8 +31,8 @@ Follow-up actions, continuous improvement planning, and iterative refinement. Th 4_REPORT → [5_FOLLOWUP] → 6_API ``` -- **Input**: Generated reports from `4_REPORT` -- **Output**: Action items, session plans, and stakeholder communications; feeds back into `1_PREPARE` +- **Intended input**: Report outcomes/follow-up specifications. +- **Intended provider-dependent output**: Action items, sessions, stakeholder communications and feedback. ## Related diff --git a/5_FOLLOWUP/execute_followup.py b/5_FOLLOWUP/execute_followup.py index b65be048..7a904ec8 100644 --- a/5_FOLLOWUP/execute_followup.py +++ b/5_FOLLOWUP/execute_followup.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime, timezone from typing import Any, Dict from data_analysis import MetricsAnalyzer @@ -13,7 +14,7 @@ FollowUpType, HTMLReportGenerator, Session, - SessionType, + SessionModality, Stakeholder, TextReportGenerator, UpdateArea, @@ -39,6 +40,9 @@ def __init__(self, specification: FollowUpSpecification): self.notification_service = NotificationService() self.database_service = DatabaseService() self.task_scheduler = TaskScheduler() + self._failed = False + self.completed_at = None + self.result = None def _setup_logger(self) -> logging.Logger: """ @@ -71,6 +75,10 @@ def execute_followup(self): ) try: + self._failed = False + self.completed_at = None + self.result = None + self.specification.generate_followup_specification() self._initialize_followup() self._execute_update_areas() self._conduct_sessions() @@ -81,17 +89,25 @@ def execute_followup(self): self._manage_timeline() self._allocate_resources() self._generate_and_distribute_reports() - self._finalize_followup() except Exception as e: + self._failed = True self.logger.error(f"Critical error during follow-up execution: {str(e)}") self.error_handler.handle_critical_error(e, self.specification.name) self.notification_service.send_alert("Critical Follow-up Error", str(e)) + raise finally: self._cleanup() - self.logger.info( - f"Completed execution of follow-up for {self.specification.name}" - ) + return self._finalize_followup() + + def _finalize_followup(self): + """Finalize only owned local state after successful operations and cleanup.""" + if self._failed: + raise RuntimeError("Follow-up contains failed operations") + self.result = self.specification.generate_followup_specification() + self.completed_at = datetime.now(timezone.utc) + self.logger.info("Completed follow-up for %s", self.specification.name) + return self.result def _initialize_followup(self): """ @@ -116,6 +132,7 @@ def _execute_update_areas(self): self.logger.info(f"Completed update for area: {area.area}") self.database_service.update_area_status(area) except Exception as e: + self._failed = True self.logger.error(f"Error updating area {area.area}: {str(e)}") area.status = "Failed" self.error_handler.handle_area_update_error(e, area) @@ -132,11 +149,11 @@ def _update_area(self, area: UpdateArea): """ # Implementation of area-specific update logic if area.area == "Database": - self.database_service.perform_database_update(area.details) + self.database_service.perform_database_update(area.notes) elif area.area == "Documentation": - self.project_manager.update_documentation(area.details) + self.project_manager.update_documentation(area.notes) elif area.area == "Infrastructure": - self.resource_allocator.update_infrastructure(area.details) + self.resource_allocator.update_infrastructure(area.notes) else: raise ValueError(f"Unsupported update area: {area.area}") @@ -149,6 +166,8 @@ def _conduct_sessions(self): for session in self.specification.sessions: self.logger.info(f"Conducting session: {session.type.name}") try: + if not isinstance(session.modality, SessionModality): + raise ValueError("Session modality must be explicitly selected") self._prepare_session(session) self._run_session(session) self._post_session_tasks(session) @@ -156,6 +175,7 @@ def _conduct_sessions(self): self.logger.info(f"Completed session: {session.type.name}") self.database_service.update_session_status(session) except Exception as e: + self._failed = True self.logger.error( f"Error conducting session {session.type.name}: {str(e)}" ) @@ -185,12 +205,12 @@ def _run_session(self, session: Session): session (Session): The session to be run. """ self.logger.info(f"Running session: {session.type.name}") - if session.type == SessionType.VIRTUAL: + if session.modality == SessionModality.VIRTUAL: self.project_manager.start_virtual_session(session) - elif session.type == SessionType.IN_PERSON: + elif session.modality == SessionModality.IN_PERSON: self.project_manager.start_in_person_session(session) else: - raise ValueError(f"Unsupported session type: {session.type}") + raise ValueError(f"Unsupported session modality: {session.modality}") def _post_session_tasks(self, session: Session): """ @@ -221,6 +241,7 @@ def _conduct_debrief(self): self.logger.info("Completed debrief session") self.database_service.update_debrief_status(debrief) except Exception as e: + self._failed = True self.logger.error(f"Error during debrief session: {str(e)}") debrief["status"] = "Failed" self.error_handler.handle_debrief_error(e, debrief) @@ -273,6 +294,7 @@ def _implement_plan(self): self.logger.info(f"{plan['type']} plan is now active") self.database_service.update_plan_status(plan) except Exception as e: + self._failed = True self.logger.error(f"Error implementing plan: {str(e)}") plan["status"] = "Failed" self.error_handler.handle_plan_implementation_error(e, plan) @@ -293,6 +315,7 @@ def _engage_stakeholders(self): ) self.database_service.update_stakeholder_engagement(stakeholder) except Exception as e: + self._failed = True self.logger.error( f"Error engaging stakeholder {stakeholder.name}: {str(e)}" ) @@ -333,6 +356,7 @@ def _track_metrics(self): self.logger.info("Updated metrics and KPIs") self.database_service.update_metrics(updated_metrics) except Exception as e: + self._failed = True self.logger.error(f"Error tracking metrics: {str(e)}") self.error_handler.handle_metrics_error(e, metrics) self.notification_service.send_alert("Metrics Tracking Failed", str(e)) @@ -351,6 +375,7 @@ def _manage_timeline(self): self.database_service.update_timeline(updated_timeline) except Exception as e: self.logger.error(f"Error managing timeline: {str(e)}") + self._failed = True self.error_handler.handle_timeline_error(e, timeline) self.notification_service.send_alert("Timeline Management Failed", str(e)) @@ -367,6 +392,7 @@ def _allocate_resources(self): self.logger.info(f"Allocated resource: {resource.name}") self.database_service.update_resource_allocation(resource) except ValueError as ve: + self._failed = True self.logger.error( f"Value error allocating resource {resource.name}: {str(ve)}" ) @@ -375,6 +401,7 @@ def _allocate_resources(self): f"Resource Allocation Failed: {resource.name}", str(ve) ) except Exception as e: + self._failed = True self.logger.error( f"Unexpected error allocating resource {resource.name}: {str(e)}" ) @@ -398,6 +425,7 @@ def _generate_and_distribute_reports(self): "Completed generation and distribution of follow-up reports" ) except Exception as e: + self._failed = True self.logger.error(f"Error generating or distributing reports: {str(e)}") self.error_handler.handle_report_error(e) @@ -440,6 +468,7 @@ def _distribute_reports(self, text_report: str, html_report: str): stakeholder.contact, "Follow-up Reports", text_report, html_report ) except Exception as e: + self._failed = True self.logger.error( f"Error sending report to {stakeholder.name}: {str(e)}" ) @@ -463,6 +492,7 @@ def _send_email( ) self.logger.info(f"Sent email to {recipient} with subject: {subject}") except Exception as e: + self._failed = True self.logger.error(f"Error sending email to {recipient}: {str(e)}") self.error_handler.handle_email_error(e, recipient) @@ -472,14 +502,22 @@ def _cleanup(self): This includes closing database connections, releasing resources, and logging the cleanup. """ self.logger.info("Performing follow-up cleanup operations") - try: - self.database_service.close_connections() - self.resource_allocator.release_allocation() - self.logger.info("Successfully performed cleanup operations") - except Exception as e: - self.logger.error(f"Error during cleanup operations: {str(e)}") - self.error_handler.handle_cleanup_error(e) - self.notification_service.send_alert("Cleanup Failed", str(e)) + first_error = None + for operation in ( + self.database_service.close_connections, + self.resource_allocator.release_allocation, + ): + try: + operation() + except Exception as exc: + self._failed = True + self.logger.error("Cleanup failed: %s", exc) + if first_error is None: + first_error = exc + if first_error is not None: + self.error_handler.handle_cleanup_error(first_error) + self.notification_service.send_alert("Cleanup Failed", str(first_error)) + raise first_error # Usage example: diff --git a/5_FOLLOWUP/specify_followup.py b/5_FOLLOWUP/specify_followup.py index 3cb98b32..717dc886 100644 --- a/5_FOLLOWUP/specify_followup.py +++ b/5_FOLLOWUP/specify_followup.py @@ -22,6 +22,26 @@ class SessionType(Enum): LESSONS_LEARNED = auto() +class SessionModality(Enum): + VIRTUAL = auto() + IN_PERSON = auto() + + +def _json_value(value): + """Encode declared enums, timestamps and durations without string fallbacks.""" + if isinstance(value, Enum): + return value.name + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, timedelta): + return value.total_seconds() + if isinstance(value, dict): + return {key: _json_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_value(item) for item in value] + return value + + @dataclass class UpdateArea: area: str @@ -37,6 +57,7 @@ class Session: date: datetime duration: timedelta facilitator: str + modality: SessionModality status: str = "Scheduled" attendees: List[str] = field(default_factory=list) outcomes: Optional[str] = None @@ -86,7 +107,7 @@ def generate(self, data: Dict[str, Any]) -> str: report_lines.append("\nSessions:") for session in data["sessions"]: report_lines.append( - f" - {session['type']} on {session['date']} for {session['duration']} minutes" + f" - {session['type']} on {session['date']} for {session['duration']} seconds" ) report_lines.append("\nDebrief Details:") @@ -140,7 +161,7 @@ def generate(self, data: Dict[str, Any]) -> str: html_content += "

Sessions

" html_content += "

Debrief Details