diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 8a5c2e5b8f..6a33bf363c 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -57,6 +57,7 @@ from .datasets import ( external, hit_testing, website, + fta_test, ) # Import explore controllers diff --git a/api/controllers/console/datasets/fta_test.py b/api/controllers/console/datasets/fta_test.py new file mode 100644 index 0000000000..e0ad370f0c --- /dev/null +++ b/api/controllers/console/datasets/fta_test.py @@ -0,0 +1,141 @@ +import json + +import requests +from sqlalchemy import text + +from controllers.console import api + +from flask_restful import Resource, reqparse +from extensions.ext_database import db + +from models.fta import ComponentFailure, ComponentFailureStats + + +class FATTestApi(Resource): + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument( + "log_process_data", + nullable=False, + required=True, + type=str, + location="args" + ) + args = parser.parse_args() + print(args["log_process_data"]) + # Extract the JSON string from the text field + json_str = args["log_process_data"].strip("```json\\n").strip("```").strip().replace("\\", "").replace(",n", ",") + log_data = json.loads(json_str) + + for data in log_data: + if not isinstance(data, dict): + raise TypeError("Data must be a dictionary.") + + required_keys = {"Date", "Component", "FailureMode", "Cause", "RepairAction", "Technician"} + if not required_keys.issubset(data.keys()): + raise ValueError(f"Data dictionary must contain the following keys: {required_keys}") + + try: + # Clear existing stats + db.session.query(ComponentFailure).delete() + component_failure = ComponentFailure( + Date=data["Date"], + Component=data["Component"], + FailureMode=data["FailureMode"], + Cause=data["Cause"], + RepairAction=data["RepairAction"], + Technician=data["Technician"] + ) + db.session.add(component_failure) + db.session.commit() + except Exception as e: + print(e) + # Clear existing stats + db.session.query(ComponentFailureStats).delete() + + # Insert calculated statistics + try: + db.session.execute( + text(""" + INSERT INTO component_failure_stats ("Component", "FailureMode", "Cause", "PossibleAction", "Probability", "MTBF") + SELECT + cf."Component", + cf."FailureMode", + cf."Cause", + cf."RepairAction" as "PossibleAction", + COUNT(*) * 1.0 / (SELECT COUNT(*) FROM component_failure WHERE "Component" = cf."Component") AS "Probability", + COALESCE(AVG(EXTRACT(EPOCH FROM (next_failure_date::timestamp - cf."Date"::timestamp)) / 86400.0),0)AS "MTBF" + FROM ( + SELECT + "Component", + "FailureMode", + "Cause", + "RepairAction", + "Date", + LEAD("Date") OVER (PARTITION BY "Component", "FailureMode", "Cause" ORDER BY "Date") AS next_failure_date + FROM + component_failure + ) cf + GROUP BY + cf."Component", cf."FailureMode", cf."Cause", cf."RepairAction"; + """) + ) + db.session.commit() + except Exception as e: + db.session.rollback() + print(f"Error during stats calculation: {e}") + # output format + # [ + # (17, 'Hydraulic system', 'Leak', 'Hose rupture', 'Replaced hydraulic hose', 0.3333333333333333, None), + # (18, 'Hydraulic system', 'Leak', 'Seal Wear', 'Replaced the faulty seal', 0.3333333333333333, None), + # (19, 'Hydraulic system', 'Pressure drop', 'Fluid leak', 'Replaced hydraulic fluid and seals', 0.3333333333333333, None) + # ] + + component_failure_stats = db.session.query(ComponentFailureStats).all() + # Convert stats to list of tuples format + stats_list = [] + for stat in component_failure_stats: + stats_list.append(( + stat.id, + stat.Component, + stat.FailureMode, + stat.Cause, + stat.PossibleAction, + stat.Probability, + stat.MTBF + )) + return {"data": stats_list}, 200 + + +# generate-fault-tree +class GenerateFaultTreeApi(Resource): + + def post(self): + parser = reqparse.RequestParser() + parser.add_argument( + "llm_text", + nullable=False, + required=True, + type=str, + location="args" + ) + args = parser.parse_args() + entities = args["llm_text"].replace("```", "").replace("\\n", "\n") + print(entities) + request_data = { + "fault_tree_text": entities + } + url = "https://fta.cognitech-dev.live/generate-fault-tree" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + response = requests.post(url, json=request_data, headers=headers) + print(response.json()) + return {"data": response.json()}, 200 + + +api.add_resource(FATTestApi, "/fta/db-handler") +api.add_resource(GenerateFaultTreeApi, "/fta/generate-fault-tree") diff --git a/api/core/tools/provider/builtin/file_extractor/tools/file_extractor.py b/api/core/tools/provider/builtin/file_extractor/tools/file_extractor.py index 3582dc2ffd..767044b0a3 100644 --- a/api/core/tools/provider/builtin/file_extractor/tools/file_extractor.py +++ b/api/core/tools/provider/builtin/file_extractor/tools/file_extractor.py @@ -1,4 +1,5 @@ from base64 import b64decode +import json import tempfile from typing import Any, Union @@ -41,7 +42,9 @@ class FileExtractorTool(BuiltinTool): embedding_model_instance=None, ) chunks = character_splitter.split_documents(documents) - return self.create_json_message(json.dumps([chunk.page_content for chunk in chunks])) + + content = "\n".join([chunk.page_content for chunk in chunks]) + return self.create_text_message(content) else: raise ToolParameterValidationError("Please provide either file") diff --git a/api/migrations/versions/2024_11_05_0326-49f175ff56cb_add_fat_test.py b/api/migrations/versions/2024_11_05_0326-49f175ff56cb_add_fat_test.py new file mode 100644 index 0000000000..935a50a0c7 --- /dev/null +++ b/api/migrations/versions/2024_11_05_0326-49f175ff56cb_add_fat_test.py @@ -0,0 +1,96 @@ +"""add_fat_test + +Revision ID: 49f175ff56cb +Revises: 43fa78bc3b7d +Create Date: 2024-11-05 03:26:22.578321 + +""" +from alembic import op +import models as models +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '49f175ff56cb' +down_revision = '09a8d1878d9b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('component_failure', + sa.Column('FailureID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('Date', sa.Date(), nullable=False), + sa.Column('Component', sa.String(length=255), nullable=False), + sa.Column('FailureMode', sa.String(length=255), nullable=False), + sa.Column('Cause', sa.String(length=255), nullable=False), + sa.Column('RepairAction', sa.Text(), nullable=True), + sa.Column('Technician', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('FailureID', name=op.f('component_failure_pkey')), + sa.UniqueConstraint('Date', 'Component', 'FailureMode', 'Cause', 'Technician', name='unique_failure_entry') + ) + op.create_table('component_failure_stats', + sa.Column('StatID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('Component', sa.String(length=255), nullable=False), + sa.Column('FailureMode', sa.String(length=255), nullable=False), + sa.Column('Cause', sa.String(length=255), nullable=False), + sa.Column('PossibleAction', sa.Text(), nullable=True), + sa.Column('Probability', sa.Float(), nullable=False), + sa.Column('MTBF', sa.Float(), nullable=False), + sa.PrimaryKeyConstraint('StatID', name=op.f('component_failure_stats_pkey')) + ) + op.create_table('incident_data', + sa.Column('IncidentID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('IncidentDescription', sa.Text(), nullable=False), + sa.Column('IncidentDate', sa.Date(), nullable=False), + sa.Column('Consequences', sa.Text(), nullable=True), + sa.Column('ResponseActions', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('IncidentID', name=op.f('incident_data_pkey')) + ) + op.create_table('maintenance', + sa.Column('MaintenanceID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('MaintenanceType', sa.String(length=255), nullable=False), + sa.Column('MaintenanceDate', sa.Date(), nullable=False), + sa.Column('ServiceDescription', sa.Text(), nullable=True), + sa.Column('PartsReplaced', sa.Text(), nullable=True), + sa.Column('Technician', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('MaintenanceID', name=op.f('maintenance_pkey')) + ) + op.create_table('operational_data', + sa.Column('OperationID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('CraneUsage', sa.Integer(), nullable=False), + sa.Column('LoadWeight', sa.Float(), nullable=False), + sa.Column('LoadFrequency', sa.Integer(), nullable=False), + sa.Column('EnvironmentalConditions', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('OperationID', name=op.f('operational_data_pkey')) + ) + op.create_table('reliability_data', + sa.Column('ComponentID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('ComponentName', sa.String(length=255), nullable=False), + sa.Column('MTBF', sa.Float(), nullable=False), + sa.Column('FailureRate', sa.Float(), nullable=False), + sa.PrimaryKeyConstraint('ComponentID', name=op.f('reliability_data_pkey')) + ) + op.create_table('safety_data', + sa.Column('SafetyID', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('SafetyInspectionDate', sa.Date(), nullable=False), + sa.Column('SafetyFindings', sa.Text(), nullable=True), + sa.Column('SafetyIncidentDescription', sa.Text(), nullable=True), + sa.Column('ComplianceStatus', sa.String(length=50), nullable=False), + sa.PrimaryKeyConstraint('SafetyID', name=op.f('safety_data_pkey')) + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('safety_data') + op.drop_table('reliability_data') + op.drop_table('operational_data') + op.drop_table('maintenance') + op.drop_table('incident_data') + op.drop_table('component_failure_stats') + op.drop_table('component_failure') + # ### end Alembic commands ### diff --git a/api/models/fta.py b/api/models/fta.py new file mode 100644 index 0000000000..ad9153f5b1 --- /dev/null +++ b/api/models/fta.py @@ -0,0 +1,100 @@ +import base64 +import enum +import hashlib +import hmac +import json +import logging +import os +import pickle +import re +import time +from json import JSONDecodeError + +from sqlalchemy import func +from sqlalchemy.dialects.postgresql import JSONB + +from configs import dify_config +from core.rag.retrieval.retrieval_methods import RetrievalMethod +from extensions.ext_database import db +from extensions.ext_storage import storage + +from .account import Account +from .model import App, Tag, TagBinding, UploadFile +from .types import StringUUID + + +class ComponentFailure(db.Model): + __tablename__ = "component_failure" + __table_args__ = ( + db.UniqueConstraint('Date', 'Component', 'FailureMode', 'Cause', 'Technician', name='unique_failure_entry'), + ) + + FailureID = db.Column(db.Integer, primary_key=True, autoincrement=True) + Date = db.Column(db.Date, nullable=False) + Component = db.Column(db.String(255), nullable=False) + FailureMode = db.Column(db.String(255), nullable=False) + Cause = db.Column(db.String(255), nullable=False) + RepairAction = db.Column(db.Text, nullable=True) + Technician = db.Column(db.String(255), nullable=False) + + +class Maintenance(db.Model): + __tablename__ = "maintenance" + + MaintenanceID = db.Column(db.Integer, primary_key=True, autoincrement=True) + MaintenanceType = db.Column(db.String(255), nullable=False) + MaintenanceDate = db.Column(db.Date, nullable=False) + ServiceDescription = db.Column(db.Text, nullable=True) + PartsReplaced = db.Column(db.Text, nullable=True) + Technician = db.Column(db.String(255), nullable=False) + + +class OperationalData(db.Model): + __tablename__ = "operational_data" + + OperationID = db.Column(db.Integer, primary_key=True, autoincrement=True) + CraneUsage = db.Column(db.Integer, nullable=False) + LoadWeight = db.Column(db.Float, nullable=False) + LoadFrequency = db.Column(db.Integer, nullable=False) + EnvironmentalConditions = db.Column(db.Text, nullable=True) + + +class IncidentData(db.Model): + __tablename__ = "incident_data" + + IncidentID = db.Column(db.Integer, primary_key=True, autoincrement=True) + IncidentDescription = db.Column(db.Text, nullable=False) + IncidentDate = db.Column(db.Date, nullable=False) + Consequences = db.Column(db.Text, nullable=True) + ResponseActions = db.Column(db.Text, nullable=True) + + +class ReliabilityData(db.Model): + __tablename__ = "reliability_data" + + ComponentID = db.Column(db.Integer, primary_key=True, autoincrement=True) + ComponentName = db.Column(db.String(255), nullable=False) + MTBF = db.Column(db.Float, nullable=False) + FailureRate = db.Column(db.Float, nullable=False) + + +class SafetyData(db.Model): + __tablename__ = "safety_data" + + SafetyID = db.Column(db.Integer, primary_key=True, autoincrement=True) + SafetyInspectionDate = db.Column(db.Date, nullable=False) + SafetyFindings = db.Column(db.Text, nullable=True) + SafetyIncidentDescription = db.Column(db.Text, nullable=True) + ComplianceStatus = db.Column(db.String(50), nullable=False) + + +class ComponentFailureStats(db.Model): + __tablename__ = "component_failure_stats" + + StatID = db.Column(db.Integer, primary_key=True, autoincrement=True) + Component = db.Column(db.String(255), nullable=False) + FailureMode = db.Column(db.String(255), nullable=False) + Cause = db.Column(db.String(255), nullable=False) + PossibleAction = db.Column(db.Text, nullable=True) + Probability = db.Column(db.Float, nullable=False) + MTBF = db.Column(db.Float, nullable=False)