From 1b1cf5fe4faa6a154e912ff0f78983584f61d00c Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 1 Dec 2025 14:36:09 +0000 Subject: [PATCH 01/40] Add db components from ispyb --- src/murfey/util/db.py | 31 +++ src/murfey/util/processing_db.py | 316 +++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 src/murfey/util/processing_db.py diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 801ac5e46..d95f5bc3e 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -9,6 +9,17 @@ import sqlalchemy from sqlmodel import Field, Relationship, SQLModel, create_engine +from murfey.util.processing_db import ( + CTF, + Atlas, + MotionCorrection, + Movie as ProcessingMovie, + ParticleClassificationGroup, + ParticlePicker, + RelativeIceThickness, + Tomogram, +) + """ GENERAL """ @@ -429,6 +440,7 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) ) + Atlas: List["Atlas"] = Relationship(back_populates="DataCollectionGroup") class NotificationParameter(SQLModel, table=True): # type: ignore @@ -468,6 +480,11 @@ class DataCollection(SQLModel, table=True): # type: ignore processing_jobs: List["ProcessingJob"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) + Movie: List["ProcessingMovie"] = Relationship(back_populates="DataCollection") + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection") class ProcessingJob(SQLModel, table=True): # type: ignore @@ -569,6 +586,20 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore murfey_ids: List["MurfeyLedger"] = Relationship( back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram") + CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram") + ParticlePicker: List["ParticlePicker"] = Relationship( + back_populates="AutoProcProgram" + ) + RelativeIceThickness: List["RelativeIceThickness"] = Relationship( + back_populates="AutoProcProgram" + ) + ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( + back_populates="AutoProcProgram" + ) class MurfeyLedger(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py new file mode 100644 index 000000000..0c9841dbe --- /dev/null +++ b/src/murfey/util/processing_db.py @@ -0,0 +1,316 @@ +import datetime +from typing import List, Optional + +from sqlmodel import Enum, Field, Relationship, SQLModel + +from murfey.util.db import AutoProcProgram, DataCollection, DataCollectionGroup + + +class Atlas(SQLModel, table=True): # type: ignore + atlasId: int = Field(primary_key=True, unique=True) + dataCollectionGroupId: int = Field(foreign_key="DataCollectionGroup.id") + atlasImage: str + pixelSize: float + cassetteSlot: Optional[int] = None + DataCollectionGroup: Optional["DataCollectionGroup"] = Relationship( + back_populates="Atlas" + ) + GridSquare: List["GridSquare"] = Relationship(back_populates="Atlas") + + +class GridSquare(SQLModel, table=True): # type: ignore + gridSquareId: int = Field(primary_key=True, unique=True) + atlasId: int = Field(foreign_key="Atlas.atlasId") + gridSquareLabel: Optional[int] = None + gridSquareImage: Optional[str] = None + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + angle: Optional[float] = None + stageLocationX: Optional[float] = None + stageLocationY: Optional[float] = None + qualityIndicator: Optional[float] = None + pixelSize: Optional[float] = None + Atlas: Optional["Atlas"] = Relationship(back_populates="Atlas") + FoilHole: List["FoilHole"] = Relationship(back_populates="GridSquare") + Tomogram: List["Tomogram"] = Relationship(back_populates="GridSquare") + + +class FoilHole(SQLModel, table=True): # type: ignore + foilHoleId: int = Field(primary_key=True, unique=True) + gridSquareId: int = Field(foreign_key="GridSquare.gridSquareId") + foilHoleLabel: str + foilHoleImage: Optional[str] = None + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + diameter: Optional[int] = None + stageLocationX: Optional[float] = None + stageLocationY: Optional[float] = None + qualityIndicator: Optional[float] = None + pixelSize: Optional[float] = None + GridSquare: Optional["GridSquare"] = Relationship(back_populates="FoilHole") + Movie: List["Movie"] = Relationship(back_populates="FoilHole") + + +class Movie(SQLModel, table=True): # type: ignore + movieId: int = Field(primary_key=True, unique=True) + createdTimeStamp: datetime.datetime + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + movieNumber: Optional[int] = None + movieFullPath: Optional[str] = None + positionX: Optional[float] = None + positionY: Optional[float] = None + nominalDefocus: Optional[float] = None + angle: Optional[float] = None + fluence: Optional[float] = None + numberOfFrames: Optional[int] = None + foilHoleId: Optional[int] = Field(foreign_key="FoilHole.foilHoleId") + templateLabel: Optional[int] = None + DataCollection: Optional["DataCollection"] = Relationship(back_populates="Movie") + FoilHole: Optional["FoilHole"] = Relationship(back_populates="Movie") + MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Movie" + ) + + +class MotionCorrection(SQLModel, table=True): # type: ignore + motionCorrectionId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + imageNumber: Optional[int] = None + firstFrame: Optional[int] = None + lastFrame: Optional[int] = None + dosePerFrame: Optional[float] = None + doseWeight: Optional[float] = None + totalMotion: Optional[float] = None + averageMotionPerFrame: Optional[float] = None + driftPlotFullPath: Optional[str] = None + micrographFullPath: Optional[str] = None + micrographSnapshotFullPath: Optional[str] = None + patchesUsedX: Optional[int] = None + patchesUsedY: Optional[int] = None + fftFullPath: Optional[str] = None + fftCorrectedFullPath: Optional[str] = None + comments: Optional[str] = None + movieId: Optional[int] = Field(foreign_key="Movie.movieId") + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="MotionCorrection" + ) + DataCollection: Optional["DataCollection"] = Relationship( + back_populates="MotionCorrection" + ) + Movie: Optional["Movie"] = Relationship(back_populates="MotionCorrection") + CTF: List["CTF"] = Relationship(back_populates="MotionCorrection") + ParticlePicker: List["ParticlePicker"] = Relationship( + back_populates="MotionCorrection" + ) + RelativeIceThickness: List["RelativeIceThickness"] = Relationship( + back_populates="MotionCorrection" + ) + + +class Tomogram(SQLModel, table=True): # type: ignore + tomogramId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + volumeFile: Optional[str] = None + stackFile: Optional[str] = None + sizeX: Optional[int] = None + sizeY: Optional[int] = None + sizeZ: Optional[int] = None + pixelSpacing: Optional[float] = None + residualErrorMean: Optional[float] = None + residualErrorSD: Optional[float] = None + xAxisCorrection: Optional[float] = None + tiltAngleOffset: Optional[float] = None + zShift: Optional[float] = None + fileDirectory: Optional[str] = None + centralSliceImage: Optional[str] = None + tomogramMovie: Optional[str] = None + xyShiftPlot: Optional[str] = None + projXY: Optional[str] = None + projXZ: Optional[str] = None + recordTimeStamp: Optional[datetime.datetime] = None + globalAlignmentQuality: Optional[float] = None + gridSquareId: Optional[int] = Field(foreign_key="GridSquare.gridSquareId") + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="Tomogram" + ) + DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") + GridSquare: Optional["GridSquare"] = Relationship(back_populates="Tomogram") + ProcessedTomogram: List["ProcessedTomogram"] = Relationship( + back_populates="Tomogram" + ) + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Tomogram" + ) + + +class CTF(SQLModel, table=True): # type: ignore + ctfId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + boxSizeX: Optional[float] = None + boxSizeY: Optional[float] = None + minResolution: Optional[float] = None + maxResolution: Optional[float] = None + minDefocus: Optional[float] = None + maxDefocus: Optional[float] = None + defocusStepSize: Optional[float] = None + astigmatism: Optional[float] = None + astigmatismAngle: Optional[float] = None + estimatedResolution: Optional[float] = None + estimatedDefocus: Optional[float] = None + amplitudeContrast: Optional[float] = None + ccValue: Optional[float] = None + fftTheoreticalFullPath: Optional[str] = None + comments: Optional[str] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship(back_populates="CTF") + MotionCorrection: Optional["MotionCorrection"] = Relationship(back_populates="CTF") + + +class ParticlePicker(SQLModel, table=True): # type: ignore + particlePickerId: int = Field(primary_key=True, unique=True) + programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + firstMotionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + particlePickingTemplate: Optional[str] = None + particleDiameter: Optional[float] = None + numberOfParticles: Optional[int] = None + summaryImageFullPath: Optional[str] = None + MotionCorrection: Optional["MotionCorrection"] = Relationship( + back_populates="ParticlePicker" + ) + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="ParticlePicker" + ) + ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( + back_populates="ParticlePicker" + ) + + +class ProcessedTomogram(SQLModel, table=True): # type: ignore + processedTomogramId: int = Field(primary_key=True, unique=True) + tomogramId: int = Field(foreign_key="Tomogram.tomogramId") + filePath: Optional[str] = None + processingType: Optional[str] = None + Tomogram: Optional["Tomogram"] = Relationship(back_populates="ProcessedTomogram") + + +class RelativeIceThickness(SQLModel, table=True): # type: ignore + relativeIceThicknessId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field( + foreign_key="AutoProcProgram.autoProcProgramId" + ) + minimum: Optional[float] = None + q1: Optional[float] = None + median: Optional[float] = None + q3: Optional[float] = None + maximum: Optional[float] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="RelativeIceThickness" + ) + MotionCorrection: Optional["MotionCorrection"] = Relationship( + back_populates="RelativeIceThickness" + ) + + +class TiltImageAlignment(SQLModel, table=True): # type: ignore + movieId: int = Field(foreign_key="Movie.movieId", primary_key=True) + tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) + defocusU: Optional[float] = None + defocusV: Optional[float] = None + psdFile: Optional[str] = None + resolution: Optional[float] = None + fitQuality: Optional[float] = None + refinedMagnification: Optional[float] = None + refinedTiltAngle: Optional[float] = None + refinedTiltAxis: Optional[float] = None + residualError: Optional[float] = None + Movie: Optional["Movie"] = Relationship(back_populates="TiltImageAlignment") + Tomogram: Optional["Tomogram"] = Relationship(back_populates="TiltImageAlignment") + + +class ParticleClassificationGroup(SQLModel, table=True): # type: ignore + particleClassificationGroupId: int = Field(primary_key=True, unique=True) + particlePickerId: Optional[int] = Field( + foreign_key="ParticlePicker.particlePickerId" + ) + programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + type: Optional[str] = Enum("2D", "3D") + batchNumber: Optional[int] = None + numberOfParticlesPerBatch: Optional[int] = None + numberOfClassesPerBatch: Optional[int] = None + symmetry: Optional[str] = None + binnedPixelSize: Optional[float] = None + ParticlePicker: Optional["ParticlePicker"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + ParticleClassification: List["ParticleClassification"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + + +class ParticleClassification(SQLModel, table=True): # type: ignore + particleClassificationId: int = Field(primary_key=True, unique=True) + classNumber: Optional[int] = None + classImageFullPath: Optional[str] = None + particlesPerClass: Optional[int] = None + rotationAccuracy: Optional[float] = None + translationAccuracy: Optional[float] = None + estimatedResolution: Optional[float] = None + overallFourierCompleteness: Optional[float] = None + particleClassificationGroupId: Optional[int] = Field( + foreign_key="ParticleClassificationGroup.particleClassificationGroupId" + ) + classDistribution: Optional[float] = None + selected: Optional[int] = None + bFactorFitIntercept: Optional[float] = None + bFactorFitLinear: Optional[float] = None + bFactorFitQuadratic: Optional[float] = None + angularEfficiency: Optional[float] = None + suggestedTilt: Optional[float] = None + CryoemInitialModel: List["CryoemInitialModel"] = Relationship( + back_populates="ParticleClassification" + ) + ParticleClassificationGroup: Optional["ParticleClassificationGroup"] = Relationship( + back_populates="ParticleClassification" + ) + BFactorFit: List["BFactorFit"] = Relationship( + back_populates="ParticleClassification" + ) + + +class BFactorFit(SQLModel, table=True): # type: ignore + bFactorFitId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="ParticleClassification.particleClassificationId" + ) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + particleBatchSize: Optional[int] = None + ParticleClassification: Optional["ParticleClassification"] = Relationship( + back_populates="BFactorFit" + ) + + +class CryoemInitialModel(SQLModel, table=True): # type: ignore + cryoemInitialModelId: int = Field(primary_key=True, unique=True) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + ParticleClassification: List["ParticleClassification"] = Relationship( + back_populates="CryoemInitialModel" + ) From ce045b685a22530fcbe779a6819a546d6a8b486e Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 1 Dec 2025 16:54:14 +0000 Subject: [PATCH 02/40] Merge together all similar tables --- src/murfey/util/db.py | 52 +++++++++++++++++++-- src/murfey/util/processing_db.py | 78 +++----------------------------- 2 files changed, 54 insertions(+), 76 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index d95f5bc3e..ea75e63bc 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -11,12 +11,11 @@ from murfey.util.processing_db import ( CTF, - Atlas, MotionCorrection, - Movie as ProcessingMovie, ParticleClassificationGroup, ParticlePicker, RelativeIceThickness, + TiltImageAlignment, Tomogram, ) @@ -440,7 +439,14 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) ) - Atlas: List["Atlas"] = Relationship(back_populates="DataCollectionGroup") + grid_squares: List["GridSquare"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) + search_maps: List["SearchMap"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) class NotificationParameter(SQLModel, table=True): # type: ignore @@ -480,7 +486,9 @@ class DataCollection(SQLModel, table=True): # type: ignore processing_jobs: List["ProcessingJob"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) - Movie: List["ProcessingMovie"] = Relationship(back_populates="DataCollection") + movies: List["Movie"] = Relationship( + back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} + ) MotionCorrection: List["MotionCorrection"] = Relationship( back_populates="DataCollection" ) @@ -639,6 +647,7 @@ class MurfeyLedger(SQLModel, table=True): # type: ignore class GridSquare(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") + atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") name: int tag: str x_location: Optional[float] @@ -650,6 +659,13 @@ class GridSquare(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None image: str = "" session: Optional[Session] = Relationship(back_populates="grid_squares") clem_image_series: List["CLEMImageSeries"] = Relationship( @@ -658,6 +674,9 @@ class GridSquare(SQLModel, table=True): # type: ignore foil_holes: List["FoilHole"] = Relationship( back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"} ) + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="grid_squares" + ) class FoilHole(SQLModel, table=True): # type: ignore @@ -674,6 +693,11 @@ class FoilHole(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + diameter: Optional[int] = None + quality_indicator: Optional[float] = None image: str = "" grid_square: Optional[GridSquare] = Relationship(back_populates="foil_holes") session: Optional[Session] = Relationship(back_populates="foil_holes") @@ -688,6 +712,7 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") + atlasId: Optional[int] = Field(foreign_key="data_collection_group.id") name: str tag: str x_location: Optional[float] = None @@ -695,6 +720,13 @@ class SearchMap(SQLModel, table=True): # type: ignore x_stage_position: Optional[float] = None y_stage_position: Optional[float] = None pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + scaled_height: Optional[int] = None + scaled_width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None image: str = "" binning: Optional[float] = None reference_matrix_m11: Optional[float] = None @@ -715,17 +747,29 @@ class SearchMap(SQLModel, table=True): # type: ignore tilt_series: List["TiltSeries"] = Relationship( back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"} ) + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="search_maps" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap") class Movie(SQLModel, table=True): # type: ignore murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id") foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None) + data_collection_id: Optional[int] = Field(foreign_key="datacollection.id") path: str image_number: int tag: str preprocessed: bool = False + createdTimeStamp: Optional[datetime] = None + movie_full_path: Optional[str] = None murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") + data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") + MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Movie" + ) class CtfParameters(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 0c9841dbe..89152d506 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -3,76 +3,7 @@ from sqlmodel import Enum, Field, Relationship, SQLModel -from murfey.util.db import AutoProcProgram, DataCollection, DataCollectionGroup - - -class Atlas(SQLModel, table=True): # type: ignore - atlasId: int = Field(primary_key=True, unique=True) - dataCollectionGroupId: int = Field(foreign_key="DataCollectionGroup.id") - atlasImage: str - pixelSize: float - cassetteSlot: Optional[int] = None - DataCollectionGroup: Optional["DataCollectionGroup"] = Relationship( - back_populates="Atlas" - ) - GridSquare: List["GridSquare"] = Relationship(back_populates="Atlas") - - -class GridSquare(SQLModel, table=True): # type: ignore - gridSquareId: int = Field(primary_key=True, unique=True) - atlasId: int = Field(foreign_key="Atlas.atlasId") - gridSquareLabel: Optional[int] = None - gridSquareImage: Optional[str] = None - pixelLocationX: Optional[int] = None - pixelLocationY: Optional[int] = None - height: Optional[int] = None - width: Optional[int] = None - angle: Optional[float] = None - stageLocationX: Optional[float] = None - stageLocationY: Optional[float] = None - qualityIndicator: Optional[float] = None - pixelSize: Optional[float] = None - Atlas: Optional["Atlas"] = Relationship(back_populates="Atlas") - FoilHole: List["FoilHole"] = Relationship(back_populates="GridSquare") - Tomogram: List["Tomogram"] = Relationship(back_populates="GridSquare") - - -class FoilHole(SQLModel, table=True): # type: ignore - foilHoleId: int = Field(primary_key=True, unique=True) - gridSquareId: int = Field(foreign_key="GridSquare.gridSquareId") - foilHoleLabel: str - foilHoleImage: Optional[str] = None - pixelLocationX: Optional[int] = None - pixelLocationY: Optional[int] = None - diameter: Optional[int] = None - stageLocationX: Optional[float] = None - stageLocationY: Optional[float] = None - qualityIndicator: Optional[float] = None - pixelSize: Optional[float] = None - GridSquare: Optional["GridSquare"] = Relationship(back_populates="FoilHole") - Movie: List["Movie"] = Relationship(back_populates="FoilHole") - - -class Movie(SQLModel, table=True): # type: ignore - movieId: int = Field(primary_key=True, unique=True) - createdTimeStamp: datetime.datetime - dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") - movieNumber: Optional[int] = None - movieFullPath: Optional[str] = None - positionX: Optional[float] = None - positionY: Optional[float] = None - nominalDefocus: Optional[float] = None - angle: Optional[float] = None - fluence: Optional[float] = None - numberOfFrames: Optional[int] = None - foilHoleId: Optional[int] = Field(foreign_key="FoilHole.foilHoleId") - templateLabel: Optional[int] = None - DataCollection: Optional["DataCollection"] = Relationship(back_populates="Movie") - FoilHole: Optional["FoilHole"] = Relationship(back_populates="Movie") - MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") - TiltImageAlignment: List["TiltImageAlignment"] = Relationship( - back_populates="Movie" - ) +from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap class MotionCorrection(SQLModel, table=True): # type: ignore @@ -134,14 +65,14 @@ class Tomogram(SQLModel, table=True): # type: ignore projXZ: Optional[str] = None recordTimeStamp: Optional[datetime.datetime] = None globalAlignmentQuality: Optional[float] = None - gridSquareId: Optional[int] = Field(foreign_key="GridSquare.gridSquareId") + gridSquareId: Optional[int] = Field(foreign_key="SearchMap.id") pixelLocationX: Optional[int] = None pixelLocationY: Optional[int] = None AutoProcProgram: Optional["AutoProcProgram"] = Relationship( back_populates="Tomogram" ) DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") - GridSquare: Optional["GridSquare"] = Relationship(back_populates="Tomogram") + SearchMap: Optional["SearchMap"] = Relationship(back_populates="Tomogram") ProcessedTomogram: List["ProcessedTomogram"] = Relationship( back_populates="Tomogram" ) @@ -309,6 +240,9 @@ class BFactorFit(SQLModel, table=True): # type: ignore class CryoemInitialModel(SQLModel, table=True): # type: ignore cryoemInitialModelId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="ParticleClassification.particleClassificationId" + ) resolution: Optional[float] = None numberOfParticles: Optional[int] = None ParticleClassification: List["ParticleClassification"] = Relationship( From e966b2a5a0134a893f816de548b6ee6d65142b10 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 2 Dec 2025 12:29:12 +0000 Subject: [PATCH 03/40] Register movies on the murfey side --- src/murfey/server/api/workflow.py | 12 +++++++++ src/murfey/server/feedback.py | 1 + src/murfey/util/db.py | 25 +++++++++---------- src/murfey/util/processing_db.py | 7 +++--- .../workflows/spa/flush_spa_preprocess.py | 1 + 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index fcc793de2..4f9744b83 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -443,6 +443,7 @@ async def request_spa_preprocessing( db.add(feedback_params) movie = Movie( murfey_id=murfey_ids[0], + data_collection_id=detached_ids[1], path=proc_file.path, image_number=proc_file.image_number, tag=proc_file.tag, @@ -695,6 +696,17 @@ async def request_tomography_preprocessing( 0 ].eer_fractionation_file + movie = Movie( + murfey_id=murfey_ids[0], + data_collection_id=dcid, + path=proc_file.path, + image_number=proc_file.image_number, + tag=proc_file.tag, + ) + db.add(movie) + db.commit() + db.close() + zocalo_message: dict = { "recipes": [recipe_name], "parameters": { diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 2e77928f3..c487b46fe 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -1488,6 +1488,7 @@ def _flush_tomography_preprocessing(message: dict, _db): p.parent.mkdir(parents=True) movie = db.Movie( murfey_id=murfey_ids[0], + data_collection_id=detached_ids[1], path=f.file_path, image_number=f.image_number, tag=f.tag, diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index ea75e63bc..3bc4a2891 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -4,20 +4,21 @@ """ from datetime import datetime -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional import sqlalchemy from sqlmodel import Field, Relationship, SQLModel, create_engine -from murfey.util.processing_db import ( - CTF, - MotionCorrection, - ParticleClassificationGroup, - ParticlePicker, - RelativeIceThickness, - TiltImageAlignment, - Tomogram, -) +if TYPE_CHECKING: + from murfey.util.processing_db import ( + CTF, + MotionCorrection, + ParticleClassificationGroup, + ParticlePicker, + RelativeIceThickness, + TiltImageAlignment, + Tomogram, + ) """ GENERAL @@ -712,7 +713,7 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlasId: Optional[int] = Field(foreign_key="data_collection_group.id") + atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") name: str tag: str x_location: Optional[float] = None @@ -761,8 +762,6 @@ class Movie(SQLModel, table=True): # type: ignore image_number: int tag: str preprocessed: bool = False - createdTimeStamp: Optional[datetime] = None - movie_full_path: Optional[str] = None murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 89152d506..5a848eb38 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -1,9 +1,10 @@ import datetime -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from sqlmodel import Enum, Field, Relationship, SQLModel -from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap +if TYPE_CHECKING: + from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap class MotionCorrection(SQLModel, table=True): # type: ignore @@ -25,7 +26,7 @@ class MotionCorrection(SQLModel, table=True): # type: ignore fftFullPath: Optional[str] = None fftCorrectedFullPath: Optional[str] = None comments: Optional[str] = None - movieId: Optional[int] = Field(foreign_key="Movie.movieId") + movieId: Optional[int] = Field(foreign_key="Movie.murfey_id") AutoProcProgram: Optional["AutoProcProgram"] = Relationship( back_populates="MotionCorrection" ) diff --git a/src/murfey/workflows/spa/flush_spa_preprocess.py b/src/murfey/workflows/spa/flush_spa_preprocess.py index 9f067f683..3cce2fdb2 100644 --- a/src/murfey/workflows/spa/flush_spa_preprocess.py +++ b/src/murfey/workflows/spa/flush_spa_preprocess.py @@ -402,6 +402,7 @@ def flush_spa_preprocess( mrcp.parent.mkdir(parents=True) movie = Movie( murfey_id=murfey_ids[2 * i], + data_collection_id=collected_ids[1].id, path=f.file_path, image_number=f.image_number, tag=f.tag, From 7dad07ebe51904ff1ded2e89e8985bd95f506360 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 2 Dec 2025 13:18:52 +0000 Subject: [PATCH 04/40] Wrong naming --- src/murfey/util/db.py | 4 ++-- src/murfey/util/processing_db.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 3bc4a2891..f1b0b9cfb 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -648,7 +648,7 @@ class MurfeyLedger(SQLModel, table=True): # type: ignore class GridSquare(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: int tag: str x_location: Optional[float] @@ -713,7 +713,7 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: str tag: str x_location: Optional[float] = None diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 5a848eb38..37209928a 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -158,7 +158,7 @@ class RelativeIceThickness(SQLModel, table=True): # type: ignore class TiltImageAlignment(SQLModel, table=True): # type: ignore - movieId: int = Field(foreign_key="Movie.movieId", primary_key=True) + movieId: int = Field(foreign_key="Movie.murfey_id", primary_key=True) tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) defocusU: Optional[float] = None defocusV: Optional[float] = None From 4027b55146c031832335cd936f8e2fd2835ca3b2 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 7 Jan 2026 09:43:53 +0000 Subject: [PATCH 05/40] Split out the dbs --- src/murfey/util/db.py | 75 +------------------- src/murfey/util/processing_db.py | 118 +++++++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 78 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index f1b0b9cfb..3f010fd04 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -4,22 +4,11 @@ """ from datetime import datetime -from typing import TYPE_CHECKING, List, Optional +from typing import List, Optional import sqlalchemy from sqlmodel import Field, Relationship, SQLModel, create_engine -if TYPE_CHECKING: - from murfey.util.processing_db import ( - CTF, - MotionCorrection, - ParticleClassificationGroup, - ParticlePicker, - RelativeIceThickness, - TiltImageAlignment, - Tomogram, - ) - """ GENERAL """ @@ -440,14 +429,6 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) ) - grid_squares: List["GridSquare"] = Relationship( - back_populates="data_collection_group", - sa_relationship_kwargs={"cascade": "delete"}, - ) - search_maps: List["SearchMap"] = Relationship( - back_populates="data_collection_group", - sa_relationship_kwargs={"cascade": "delete"}, - ) class NotificationParameter(SQLModel, table=True): # type: ignore @@ -487,13 +468,6 @@ class DataCollection(SQLModel, table=True): # type: ignore processing_jobs: List["ProcessingJob"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) - movies: List["Movie"] = Relationship( - back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} - ) - MotionCorrection: List["MotionCorrection"] = Relationship( - back_populates="DataCollection" - ) - Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection") class ProcessingJob(SQLModel, table=True): # type: ignore @@ -595,20 +569,6 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore murfey_ids: List["MurfeyLedger"] = Relationship( back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) - MotionCorrection: List["MotionCorrection"] = Relationship( - back_populates="DataCollection" - ) - Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram") - CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram") - ParticlePicker: List["ParticlePicker"] = Relationship( - back_populates="AutoProcProgram" - ) - RelativeIceThickness: List["RelativeIceThickness"] = Relationship( - back_populates="AutoProcProgram" - ) - ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( - back_populates="AutoProcProgram" - ) class MurfeyLedger(SQLModel, table=True): # type: ignore @@ -660,13 +620,6 @@ class GridSquare(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None - scaled_pixel_size: Optional[float] = None - pixel_location_x: Optional[int] = None - pixel_location_y: Optional[int] = None - height: Optional[int] = None - width: Optional[int] = None - angle: Optional[float] = None - quality_indicator: Optional[float] = None image: str = "" session: Optional[Session] = Relationship(back_populates="grid_squares") clem_image_series: List["CLEMImageSeries"] = Relationship( @@ -675,9 +628,6 @@ class GridSquare(SQLModel, table=True): # type: ignore foil_holes: List["FoilHole"] = Relationship( back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"} ) - data_collection_group: Optional["DataCollectionGroup"] = Relationship( - back_populates="grid_squares" - ) class FoilHole(SQLModel, table=True): # type: ignore @@ -694,11 +644,6 @@ class FoilHole(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None - scaled_pixel_size: Optional[float] = None - pixel_location_x: Optional[int] = None - pixel_location_y: Optional[int] = None - diameter: Optional[int] = None - quality_indicator: Optional[float] = None image: str = "" grid_square: Optional[GridSquare] = Relationship(back_populates="foil_holes") session: Optional[Session] = Relationship(back_populates="foil_holes") @@ -713,7 +658,6 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: str tag: str x_location: Optional[float] = None @@ -721,13 +665,6 @@ class SearchMap(SQLModel, table=True): # type: ignore x_stage_position: Optional[float] = None y_stage_position: Optional[float] = None pixel_size: Optional[float] = None - scaled_pixel_size: Optional[float] = None - pixel_location_x: Optional[int] = None - pixel_location_y: Optional[int] = None - scaled_height: Optional[int] = None - scaled_width: Optional[int] = None - angle: Optional[float] = None - quality_indicator: Optional[float] = None image: str = "" binning: Optional[float] = None reference_matrix_m11: Optional[float] = None @@ -748,27 +685,17 @@ class SearchMap(SQLModel, table=True): # type: ignore tilt_series: List["TiltSeries"] = Relationship( back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"} ) - data_collection_group: Optional["DataCollectionGroup"] = Relationship( - back_populates="search_maps" - ) - Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap") class Movie(SQLModel, table=True): # type: ignore murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id") foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None) - data_collection_id: Optional[int] = Field(foreign_key="datacollection.id") path: str image_number: int tag: str preprocessed: bool = False murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") - data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") - MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") - TiltImageAlignment: List["TiltImageAlignment"] = Relationship( - back_populates="Movie" - ) class CtfParameters(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 37209928a..df233dd83 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -1,10 +1,102 @@ import datetime -from typing import TYPE_CHECKING, List, Optional +from typing import List, Optional -from sqlmodel import Enum, Field, Relationship, SQLModel +import sqlalchemy +from sqlmodel import Enum, Field, Relationship, create_engine -if TYPE_CHECKING: - from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap +from murfey.util.db import ( + AutoProcProgram as AutoProcProgramOrig, + DataCollection as DataCollectionOrig, + DataCollectionGroup as DataCollectionGroupOrig, + FoilHole as FoilHoleOrig, + GridSquare as GridSquareOrig, + Movie as MovieOrig, + SearchMap as SearchMapOrig, + SQLModel, +) + + +class DataCollectionGroup(DataCollectionGroupOrig): + grid_squares: List["GridSquare"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) + search_maps: List["SearchMap"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) + + +class DataCollection(DataCollectionOrig): + movies: List["Movie"] = Relationship( + back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} + ) + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection") + + +class AutoProcProgram(AutoProcProgramOrig): + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram") + CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram") + ParticlePicker: List["ParticlePicker"] = Relationship( + back_populates="AutoProcProgram" + ) + RelativeIceThickness: List["RelativeIceThickness"] = Relationship( + back_populates="AutoProcProgram" + ) + ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( + back_populates="AutoProcProgram" + ) + + +class GridSquare(GridSquareOrig): + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="grid_squares" + ) + + +class FoilHole(FoilHoleOrig): + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + diameter: Optional[int] = None + quality_indicator: Optional[float] = None + + +class SearchMap(SearchMapOrig): + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + scaled_height: Optional[int] = None + scaled_width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="search_maps" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap") + + +class Movie(MovieOrig): + data_collection_id: Optional[int] = Field(foreign_key="datacollection.id") + data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") + MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Movie" + ) class MotionCorrection(SQLModel, table=True): # type: ignore @@ -249,3 +341,21 @@ class CryoemInitialModel(SQLModel, table=True): # type: ignore ParticleClassification: List["ParticleClassification"] = Relationship( back_populates="CryoemInitialModel" ) + + +""" +FUNCTIONS +""" + + +def setup(url: str): + engine = create_engine(url) + SQLModel.metadata.create_all(engine) + + +def clear(url: str): + engine = create_engine(url) + metadata = sqlalchemy.MetaData() + metadata.create_all(engine) + metadata.reflect(engine) + metadata.drop_all(engine) From 3bc60aaffd92736b5bc4cdd967e140d186d3535b Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 8 Jan 2026 14:57:20 +0000 Subject: [PATCH 06/40] Columns different way around --- src/murfey/util/db.py | 3 ++- src/murfey/util/processing_db.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 3f010fd04..226122096 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -608,7 +608,6 @@ class MurfeyLedger(SQLModel, table=True): # type: ignore class GridSquare(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: int tag: str x_location: Optional[float] @@ -689,12 +688,14 @@ class SearchMap(SQLModel, table=True): # type: ignore class Movie(SQLModel, table=True): # type: ignore murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id") + data_collection_id: Optional[int] = Field(foreign_key="datacollection.id") foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None) path: str image_number: int tag: str preprocessed: bool = False murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") + data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index df233dd83..d22953ea0 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -55,6 +55,7 @@ class AutoProcProgram(AutoProcProgramOrig): class GridSquare(GridSquareOrig): + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") scaled_pixel_size: Optional[float] = None pixel_location_x: Optional[int] = None pixel_location_y: Optional[int] = None @@ -91,8 +92,6 @@ class SearchMap(SearchMapOrig): class Movie(MovieOrig): - data_collection_id: Optional[int] = Field(foreign_key="datacollection.id") - data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") TiltImageAlignment: List["TiltImageAlignment"] = Relationship( back_populates="Movie" From 414ecdde13c695b5c220991b0faedbb1028e8cf4 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 8 Jan 2026 15:06:38 +0000 Subject: [PATCH 07/40] Got mappings wrong --- src/murfey/util/db.py | 3 +++ src/murfey/util/processing_db.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 226122096..e379e677a 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -468,6 +468,9 @@ class DataCollection(SQLModel, table=True): # type: ignore processing_jobs: List["ProcessingJob"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) + movies: List["Movie"] = Relationship( + back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} + ) class ProcessingJob(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index d22953ea0..365e90641 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -28,9 +28,6 @@ class DataCollectionGroup(DataCollectionGroupOrig): class DataCollection(DataCollectionOrig): - movies: List["Movie"] = Relationship( - back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} - ) MotionCorrection: List["MotionCorrection"] = Relationship( back_populates="DataCollection" ) From d5fe52b7160ef03a13193f567897fc3fc7c00c32 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 9 Jan 2026 10:23:12 +0000 Subject: [PATCH 08/40] Add setup options for processing tables --- src/murfey/cli/create_db.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/murfey/cli/create_db.py b/src/murfey/cli/create_db.py index d63470ac0..e1add0683 100644 --- a/src/murfey/cli/create_db.py +++ b/src/murfey/cli/create_db.py @@ -1,13 +1,16 @@ import argparse from murfey.util.db import clear, setup +from murfey.util.processing_db import ( + clear as processing_db_clear, + setup as processing_db_setup, +) def run(): parser = argparse.ArgumentParser( description="Generate the necessary tables for the Murfey database" ) - parser.add_argument( "--no-clear", dest="clear", @@ -15,11 +18,24 @@ def run(): action="store_false", help="Do not clear current database tables before creating specified tables", ) + parser.add_argument( + "--include-processing", + dest="processing", + default=True, + action="store_true", + help="Include processing results tables (MotionCorr, CTF, etc)", + ) args = parser.parse_args() from murfey.server.murfey_db import url - if args.clear: + if args.clear and args.processing: + processing_db_clear(url()) + elif args.clear: clear(url()) - setup(url()) + + if args.processing: + processing_db_setup(url()) + else: + setup(url()) From 70ebca90037c694afbcd249c8637c7d3432e1941 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 26 Jan 2026 13:35:35 +0000 Subject: [PATCH 09/40] Test that the processing db can be set up --- src/murfey/util/processing_db.py | 78 ++++++++++++++++---------------- tests/util/test_processing_db.py | 71 +++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 39 deletions(-) create mode 100644 tests/util/test_processing_db.py diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 365e90641..439e54b7f 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -131,45 +131,6 @@ class MotionCorrection(SQLModel, table=True): # type: ignore ) -class Tomogram(SQLModel, table=True): # type: ignore - tomogramId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") - volumeFile: Optional[str] = None - stackFile: Optional[str] = None - sizeX: Optional[int] = None - sizeY: Optional[int] = None - sizeZ: Optional[int] = None - pixelSpacing: Optional[float] = None - residualErrorMean: Optional[float] = None - residualErrorSD: Optional[float] = None - xAxisCorrection: Optional[float] = None - tiltAngleOffset: Optional[float] = None - zShift: Optional[float] = None - fileDirectory: Optional[str] = None - centralSliceImage: Optional[str] = None - tomogramMovie: Optional[str] = None - xyShiftPlot: Optional[str] = None - projXY: Optional[str] = None - projXZ: Optional[str] = None - recordTimeStamp: Optional[datetime.datetime] = None - globalAlignmentQuality: Optional[float] = None - gridSquareId: Optional[int] = Field(foreign_key="SearchMap.id") - pixelLocationX: Optional[int] = None - pixelLocationY: Optional[int] = None - AutoProcProgram: Optional["AutoProcProgram"] = Relationship( - back_populates="Tomogram" - ) - DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") - SearchMap: Optional["SearchMap"] = Relationship(back_populates="Tomogram") - ProcessedTomogram: List["ProcessedTomogram"] = Relationship( - back_populates="Tomogram" - ) - TiltImageAlignment: List["TiltImageAlignment"] = Relationship( - back_populates="Tomogram" - ) - - class CTF(SQLModel, table=True): # type: ignore ctfId: int = Field(primary_key=True, unique=True) motionCorrectionId: Optional[int] = Field( @@ -216,6 +177,45 @@ class ParticlePicker(SQLModel, table=True): # type: ignore ) +class Tomogram(SQLModel, table=True): # type: ignore + tomogramId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + volumeFile: Optional[str] = None + stackFile: Optional[str] = None + sizeX: Optional[int] = None + sizeY: Optional[int] = None + sizeZ: Optional[int] = None + pixelSpacing: Optional[float] = None + residualErrorMean: Optional[float] = None + residualErrorSD: Optional[float] = None + xAxisCorrection: Optional[float] = None + tiltAngleOffset: Optional[float] = None + zShift: Optional[float] = None + fileDirectory: Optional[str] = None + centralSliceImage: Optional[str] = None + tomogramMovie: Optional[str] = None + xyShiftPlot: Optional[str] = None + projXY: Optional[str] = None + projXZ: Optional[str] = None + recordTimeStamp: Optional[datetime.datetime] = None + globalAlignmentQuality: Optional[float] = None + gridSquareId: Optional[int] = Field(foreign_key="SearchMap.id") + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="Tomogram" + ) + DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") + SearchMap: Optional["SearchMap"] = Relationship(back_populates="Tomogram") + ProcessedTomogram: List["ProcessedTomogram"] = Relationship( + back_populates="Tomogram" + ) + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Tomogram" + ) + + class ProcessedTomogram(SQLModel, table=True): # type: ignore processedTomogramId: int = Field(primary_key=True, unique=True) tomogramId: int = Field(foreign_key="Tomogram.tomogramId") diff --git a/tests/util/test_processing_db.py b/tests/util/test_processing_db.py new file mode 100644 index 000000000..528e4159b --- /dev/null +++ b/tests/util/test_processing_db.py @@ -0,0 +1,71 @@ +import os + +import pytest +from sqlalchemy import MetaData, create_engine +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, select + +from murfey.util.processing_db import ( + CTF, + BFactorFit, + CryoemInitialModel, + DataCollectionGroup, + MotionCorrection, + ParticleClassification, + ParticleClassificationGroup, + ParticlePicker, + ProcessedTomogram, + RelativeIceThickness, + TiltImageAlignment, + Tomogram, +) + + +@pytest.fixture(scope="session") +def murfey_db_url(): + try: + return ( + f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}" + f"@{os.environ['POSTGRES_HOST']}:{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_DB']}" + ) + # Skip Murfey database-related tests if the environment for it hasn't been set up + except KeyError: + # If this fails in the GitHub test environment, raise it as a genuine error + if os.getenv("GITHUB_ACTIONS") == "true": + raise KeyError + pytest.skip("Murfey PostgreSQL database has not been set up; skipping test") + return "" + + +def test_processing_tables_exist(murfey_db_url): + from sqlmodel import SQLModel + + engine = create_engine(murfey_db_url) + SQLModel.metadata.create_all(engine) + connection = engine.connect() + + with sessionmaker( + bind=connection, expire_on_commit=False, class_=Session + ) as murfey_db_session: + assert murfey_db_session.exec(select(DataCollectionGroup)).all() == [] + assert murfey_db_session.exec(select(MotionCorrection)).all() == [] + assert murfey_db_session.exec(select(CTF)).all() == [] + assert murfey_db_session.exec(select(ParticlePicker)).all() == [] + assert murfey_db_session.exec(select(Tomogram)).all() == [] + assert murfey_db_session.exec(select(ProcessedTomogram)).all() == [] + assert murfey_db_session.exec(select(RelativeIceThickness)).all() == [] + assert murfey_db_session.exec(select(TiltImageAlignment)).all() == [] + assert murfey_db_session.exec(select(ParticleClassificationGroup)).all() == [] + assert murfey_db_session.exec(select(ParticleClassification)).all() == [] + assert murfey_db_session.exec(select(BFactorFit)).all() == [] + assert murfey_db_session.exec(select(CryoemInitialModel)).all() == [] + + from murfey.util.db import ClientEnvironment + + assert murfey_db_session.exec(select(ClientEnvironment)).all() == [] + + connection.close() + metadata = MetaData() + metadata.create_all(engine) + metadata.reflect(engine) + metadata.drop_all(engine) From c5cf4154dbf387a8da3028973ac6dbcc2576588f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 26 Jan 2026 13:47:00 +0000 Subject: [PATCH 10/40] Attempt model config --- src/murfey/util/processing_db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 439e54b7f..fd4fbad24 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -2,6 +2,7 @@ from typing import List, Optional import sqlalchemy +from pydantic import ConfigDict from sqlmodel import Enum, Field, Relationship, create_engine from murfey.util.db import ( @@ -26,6 +27,8 @@ class DataCollectionGroup(DataCollectionGroupOrig): sa_relationship_kwargs={"cascade": "delete"}, ) + model_config = ConfigDict(arbitrary_types_allowed=True) + class DataCollection(DataCollectionOrig): MotionCorrection: List["MotionCorrection"] = Relationship( From 8bcb49f993bed329ee8126f8f6d7c477162072a8 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 26 Jan 2026 13:57:45 +0000 Subject: [PATCH 11/40] Try different relation method --- src/murfey/util/db.py | 2 +- src/murfey/util/processing_db.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 4fe0fa9ac..7a6fb000f 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -414,7 +414,7 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore atlas_pixel_size: Optional[float] = None atlas: str = "" sample: Optional[int] = None - session: Optional[Session] = Relationship(back_populates="data_collection_groups") + session: Optional["Session"] = Relationship(back_populates="data_collection_groups") data_collections: List["DataCollection"] = Relationship( back_populates="data_collection_group", sa_relationship_kwargs={"cascade": "delete"}, diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index fd4fbad24..439e54b7f 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -2,7 +2,6 @@ from typing import List, Optional import sqlalchemy -from pydantic import ConfigDict from sqlmodel import Enum, Field, Relationship, create_engine from murfey.util.db import ( @@ -27,8 +26,6 @@ class DataCollectionGroup(DataCollectionGroupOrig): sa_relationship_kwargs={"cascade": "delete"}, ) - model_config = ConfigDict(arbitrary_types_allowed=True) - class DataCollection(DataCollectionOrig): MotionCorrection: List["MotionCorrection"] = Relationship( From 4d87a9764454bf38d3aa9c909771b193fafd94d0 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 26 Jan 2026 14:01:15 +0000 Subject: [PATCH 12/40] Back to arb types --- src/murfey/util/processing_db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 439e54b7f..fd4fbad24 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -2,6 +2,7 @@ from typing import List, Optional import sqlalchemy +from pydantic import ConfigDict from sqlmodel import Enum, Field, Relationship, create_engine from murfey.util.db import ( @@ -26,6 +27,8 @@ class DataCollectionGroup(DataCollectionGroupOrig): sa_relationship_kwargs={"cascade": "delete"}, ) + model_config = ConfigDict(arbitrary_types_allowed=True) + class DataCollection(DataCollectionOrig): MotionCorrection: List["MotionCorrection"] = Relationship( From 3e67f3d427a98409003dfe6abce7e54c7baab880 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 19 Feb 2026 16:58:24 +0000 Subject: [PATCH 13/40] Try importing whole db --- src/murfey/util/processing_db.py | 53 +++++++++++++------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index fd4fbad24..42cd66be8 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -5,19 +5,10 @@ from pydantic import ConfigDict from sqlmodel import Enum, Field, Relationship, create_engine -from murfey.util.db import ( - AutoProcProgram as AutoProcProgramOrig, - DataCollection as DataCollectionOrig, - DataCollectionGroup as DataCollectionGroupOrig, - FoilHole as FoilHoleOrig, - GridSquare as GridSquareOrig, - Movie as MovieOrig, - SearchMap as SearchMapOrig, - SQLModel, -) - - -class DataCollectionGroup(DataCollectionGroupOrig): +from murfey.util import db + + +class DataCollectionGroup(db.DataCollectionGroup): grid_squares: List["GridSquare"] = Relationship( back_populates="data_collection_group", sa_relationship_kwargs={"cascade": "delete"}, @@ -30,14 +21,14 @@ class DataCollectionGroup(DataCollectionGroupOrig): model_config = ConfigDict(arbitrary_types_allowed=True) -class DataCollection(DataCollectionOrig): +class DataCollection(db.DataCollection): MotionCorrection: List["MotionCorrection"] = Relationship( back_populates="DataCollection" ) Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection") -class AutoProcProgram(AutoProcProgramOrig): +class AutoProcProgram(db.AutoProcProgram): MotionCorrection: List["MotionCorrection"] = Relationship( back_populates="DataCollection" ) @@ -54,7 +45,7 @@ class AutoProcProgram(AutoProcProgramOrig): ) -class GridSquare(GridSquareOrig): +class GridSquare(db.GridSquare): atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") scaled_pixel_size: Optional[float] = None pixel_location_x: Optional[int] = None @@ -68,7 +59,7 @@ class GridSquare(GridSquareOrig): ) -class FoilHole(FoilHoleOrig): +class FoilHole(db.FoilHole): scaled_pixel_size: Optional[float] = None pixel_location_x: Optional[int] = None pixel_location_y: Optional[int] = None @@ -76,7 +67,7 @@ class FoilHole(FoilHoleOrig): quality_indicator: Optional[float] = None -class SearchMap(SearchMapOrig): +class SearchMap(db.SearchMap): atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") scaled_pixel_size: Optional[float] = None pixel_location_x: Optional[int] = None @@ -91,14 +82,14 @@ class SearchMap(SearchMapOrig): Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap") -class Movie(MovieOrig): +class Movie(db.Movie): MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") TiltImageAlignment: List["TiltImageAlignment"] = Relationship( back_populates="Movie" ) -class MotionCorrection(SQLModel, table=True): # type: ignore +class MotionCorrection(db.SQLModel, table=True): # type: ignore motionCorrectionId: int = Field(primary_key=True, unique=True) dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") @@ -134,7 +125,7 @@ class MotionCorrection(SQLModel, table=True): # type: ignore ) -class CTF(SQLModel, table=True): # type: ignore +class CTF(db.SQLModel, table=True): # type: ignore ctfId: int = Field(primary_key=True, unique=True) motionCorrectionId: Optional[int] = Field( foreign_key="MotionCorrection.motionCorrectionId" @@ -159,7 +150,7 @@ class CTF(SQLModel, table=True): # type: ignore MotionCorrection: Optional["MotionCorrection"] = Relationship(back_populates="CTF") -class ParticlePicker(SQLModel, table=True): # type: ignore +class ParticlePicker(db.SQLModel, table=True): # type: ignore particlePickerId: int = Field(primary_key=True, unique=True) programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") firstMotionCorrectionId: Optional[int] = Field( @@ -180,7 +171,7 @@ class ParticlePicker(SQLModel, table=True): # type: ignore ) -class Tomogram(SQLModel, table=True): # type: ignore +class Tomogram(db.SQLModel, table=True): # type: ignore tomogramId: int = Field(primary_key=True, unique=True) dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") @@ -219,7 +210,7 @@ class Tomogram(SQLModel, table=True): # type: ignore ) -class ProcessedTomogram(SQLModel, table=True): # type: ignore +class ProcessedTomogram(db.SQLModel, table=True): # type: ignore processedTomogramId: int = Field(primary_key=True, unique=True) tomogramId: int = Field(foreign_key="Tomogram.tomogramId") filePath: Optional[str] = None @@ -227,7 +218,7 @@ class ProcessedTomogram(SQLModel, table=True): # type: ignore Tomogram: Optional["Tomogram"] = Relationship(back_populates="ProcessedTomogram") -class RelativeIceThickness(SQLModel, table=True): # type: ignore +class RelativeIceThickness(db.SQLModel, table=True): # type: ignore relativeIceThicknessId: int = Field(primary_key=True, unique=True) motionCorrectionId: Optional[int] = Field( foreign_key="MotionCorrection.motionCorrectionId" @@ -248,7 +239,7 @@ class RelativeIceThickness(SQLModel, table=True): # type: ignore ) -class TiltImageAlignment(SQLModel, table=True): # type: ignore +class TiltImageAlignment(db.SQLModel, table=True): # type: ignore movieId: int = Field(foreign_key="Movie.murfey_id", primary_key=True) tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) defocusU: Optional[float] = None @@ -264,7 +255,7 @@ class TiltImageAlignment(SQLModel, table=True): # type: ignore Tomogram: Optional["Tomogram"] = Relationship(back_populates="TiltImageAlignment") -class ParticleClassificationGroup(SQLModel, table=True): # type: ignore +class ParticleClassificationGroup(db.SQLModel, table=True): # type: ignore particleClassificationGroupId: int = Field(primary_key=True, unique=True) particlePickerId: Optional[int] = Field( foreign_key="ParticlePicker.particlePickerId" @@ -287,7 +278,7 @@ class ParticleClassificationGroup(SQLModel, table=True): # type: ignore ) -class ParticleClassification(SQLModel, table=True): # type: ignore +class ParticleClassification(db.SQLModel, table=True): # type: ignore particleClassificationId: int = Field(primary_key=True, unique=True) classNumber: Optional[int] = None classImageFullPath: Optional[str] = None @@ -317,7 +308,7 @@ class ParticleClassification(SQLModel, table=True): # type: ignore ) -class BFactorFit(SQLModel, table=True): # type: ignore +class BFactorFit(db.SQLModel, table=True): # type: ignore bFactorFitId: int = Field(primary_key=True, unique=True) particleClassificationId: int = Field( foreign_key="ParticleClassification.particleClassificationId" @@ -330,7 +321,7 @@ class BFactorFit(SQLModel, table=True): # type: ignore ) -class CryoemInitialModel(SQLModel, table=True): # type: ignore +class CryoemInitialModel(db.SQLModel, table=True): # type: ignore cryoemInitialModelId: int = Field(primary_key=True, unique=True) particleClassificationId: int = Field( foreign_key="ParticleClassification.particleClassificationId" @@ -349,7 +340,7 @@ class CryoemInitialModel(SQLModel, table=True): # type: ignore def setup(url: str): engine = create_engine(url) - SQLModel.metadata.create_all(engine) + db.SQLModel.metadata.create_all(engine) def clear(url: str): From 5e2003ab07429a3a8f1c26571f4c8043fe68d0ed Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 19 Feb 2026 17:04:59 +0000 Subject: [PATCH 14/40] Import and pass --- src/murfey/util/processing_db.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 42cd66be8..cfbb11c5c 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -8,6 +8,22 @@ from murfey.util import db +class CLEMImageSeries(db.CLEMImageSeries): + pass + + +class NotificationParameter(db.NotificationParameter): + pass + + +class Session(db.Session): + pass + + +class TomographyProcessingParameters(db.TomographyProcessingParameters): + pass + + class DataCollectionGroup(db.DataCollectionGroup): grid_squares: List["GridSquare"] = Relationship( back_populates="data_collection_group", From ace3aba19373bcba9d919c821e35da1b9ec0a6f8 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 19 Feb 2026 17:10:21 +0000 Subject: [PATCH 15/40] Arbitrary types --- src/murfey/util/processing_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index cfbb11c5c..acff49301 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -9,7 +9,7 @@ class CLEMImageSeries(db.CLEMImageSeries): - pass + model_config = ConfigDict(arbitrary_types_allowed=True) class NotificationParameter(db.NotificationParameter): From 4f118ee8af8bbde45e3e361174b0d00056167cad Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 19 Feb 2026 17:19:07 +0000 Subject: [PATCH 16/40] More of the chain --- src/murfey/util/processing_db.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index acff49301..345de799c 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -9,7 +9,19 @@ class CLEMImageSeries(db.CLEMImageSeries): - model_config = ConfigDict(arbitrary_types_allowed=True) + pass + + +class CLEMLIFFile(db.CLEMLIFFile): + pass + + +class CLEMImageMetadata(db.CLEMImageMetadata): + pass + + +class CLEMImageStack(db.CLEMImageStack): + pass class NotificationParameter(db.NotificationParameter): From 1a6ba4b9bd1ce55831a47bc8f4156048cff9c70f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 09:09:49 +0000 Subject: [PATCH 17/40] keep going --- src/murfey/util/processing_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 345de799c..fb2f31a03 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -9,7 +9,7 @@ class CLEMImageSeries(db.CLEMImageSeries): - pass + model_config = ConfigDict(arbitrary_types_allowed=True) class CLEMLIFFile(db.CLEMLIFFile): From d6c752b5fec0ae028079c21f926c28f755961c10 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 09:17:47 +0000 Subject: [PATCH 18/40] make table --- src/murfey/util/db.py | 5 ++++- src/murfey/util/processing_db.py | 33 +------------------------------- 2 files changed, 5 insertions(+), 33 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 7a6fb000f..8d421ff8b 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -406,7 +406,7 @@ class Tilt(SQLModel, table=True): # type: ignore tilt_series: Optional[TiltSeries] = Relationship(back_populates="tilts") -class DataCollectionGroup(SQLModel, table=True): # type: ignore +class DataCollectionGroupModel(SQLModel): # type: ignore id: int = Field(primary_key=True, unique=True) session_id: int = Field(foreign_key="session.id", primary_key=True) tag: str = Field(primary_key=True) @@ -435,6 +435,9 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore ) +class DataCollectionGroup(DataCollectionGroupModel, table=True): + pass + class NotificationParameter(SQLModel, table=True): # type: ignore id: Optional[int] = Field(default=None, primary_key=True) dcg_id: int = Field(foreign_key="datacollectiongroup.id") diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index fb2f31a03..108b4b50b 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -2,41 +2,12 @@ from typing import List, Optional import sqlalchemy -from pydantic import ConfigDict from sqlmodel import Enum, Field, Relationship, create_engine from murfey.util import db -class CLEMImageSeries(db.CLEMImageSeries): - model_config = ConfigDict(arbitrary_types_allowed=True) - - -class CLEMLIFFile(db.CLEMLIFFile): - pass - - -class CLEMImageMetadata(db.CLEMImageMetadata): - pass - - -class CLEMImageStack(db.CLEMImageStack): - pass - - -class NotificationParameter(db.NotificationParameter): - pass - - -class Session(db.Session): - pass - - -class TomographyProcessingParameters(db.TomographyProcessingParameters): - pass - - -class DataCollectionGroup(db.DataCollectionGroup): +class DataCollectionGroup(db.DataCollectionGroupModel, table=True): grid_squares: List["GridSquare"] = Relationship( back_populates="data_collection_group", sa_relationship_kwargs={"cascade": "delete"}, @@ -46,8 +17,6 @@ class DataCollectionGroup(db.DataCollectionGroup): sa_relationship_kwargs={"cascade": "delete"}, ) - model_config = ConfigDict(arbitrary_types_allowed=True) - class DataCollection(db.DataCollection): MotionCorrection: List["MotionCorrection"] = Relationship( From c0c686f2d03358afc00a04c063c554e8e61f1ae5 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 09:20:59 +0000 Subject: [PATCH 19/40] ignore types --- src/murfey/util/db.py | 3 ++- src/murfey/util/processing_db.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 8d421ff8b..8a983f14f 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -435,9 +435,10 @@ class DataCollectionGroupModel(SQLModel): # type: ignore ) -class DataCollectionGroup(DataCollectionGroupModel, table=True): +class DataCollectionGroup(DataCollectionGroupModel, table=True): # type: ignore pass + class NotificationParameter(SQLModel, table=True): # type: ignore id: Optional[int] = Field(default=None, primary_key=True) dcg_id: int = Field(foreign_key="datacollectiongroup.id") diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 108b4b50b..af5005465 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -7,7 +7,7 @@ from murfey.util import db -class DataCollectionGroup(db.DataCollectionGroupModel, table=True): +class DataCollectionGroup(db.DataCollectionGroupModel, table=True): # type: ignore grid_squares: List["GridSquare"] = Relationship( back_populates="data_collection_group", sa_relationship_kwargs={"cascade": "delete"}, From 8ac9fe2ae4fe33dcf95e106a216d97c54ab9c03f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 09:34:07 +0000 Subject: [PATCH 20/40] mapping --- src/murfey/util/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 8a983f14f..1431046fb 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -12,6 +12,7 @@ """ GENERAL """ +mapper_registry = sqlalchemy.orm.registry() class MurfeyUser(SQLModel, table=True): # type: ignore @@ -435,6 +436,7 @@ class DataCollectionGroupModel(SQLModel): # type: ignore ) +@mapper_registry.mapped class DataCollectionGroup(DataCollectionGroupModel, table=True): # type: ignore pass From 9575c34eea20cc7161f4302bb0012ae32516d780 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 09:39:53 +0000 Subject: [PATCH 21/40] table inherit table --- src/murfey/util/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 1431046fb..dc551bfb0 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -407,7 +407,7 @@ class Tilt(SQLModel, table=True): # type: ignore tilt_series: Optional[TiltSeries] = Relationship(back_populates="tilts") -class DataCollectionGroupModel(SQLModel): # type: ignore +class DataCollectionGroupModel(SQLModel, table=True): # type: ignore id: int = Field(primary_key=True, unique=True) session_id: int = Field(foreign_key="session.id", primary_key=True) tag: str = Field(primary_key=True) From 9e2e595adc2fe4a37eb527d8acd2191c3b296e0b Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 10:25:05 +0000 Subject: [PATCH 22/40] Refactor for no inheritance --- src/murfey/util/db.py | 84 ++++++++++++-- src/murfey/util/processing_db.py | 181 +++++++++---------------------- 2 files changed, 130 insertions(+), 135 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index dc551bfb0..1beb9e7b2 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -4,11 +4,22 @@ """ from datetime import datetime -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional import sqlalchemy from sqlmodel import Field, Relationship, SQLModel, create_engine +if TYPE_CHECKING: + from murfey.util.processing_db import ( + CTF, + MotionCorrection, + ParticleClassificationGroup, + ParticlePicker, + RelativeIceThickness, + TiltImageAlignment, + Tomogram, + ) + """ GENERAL """ @@ -407,7 +418,7 @@ class Tilt(SQLModel, table=True): # type: ignore tilt_series: Optional[TiltSeries] = Relationship(back_populates="tilts") -class DataCollectionGroupModel(SQLModel, table=True): # type: ignore +class DataCollectionGroup(SQLModel, table=True): # type: ignore id: int = Field(primary_key=True, unique=True) session_id: int = Field(foreign_key="session.id", primary_key=True) tag: str = Field(primary_key=True) @@ -434,11 +445,14 @@ class DataCollectionGroupModel(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) ) - - -@mapper_registry.mapped -class DataCollectionGroup(DataCollectionGroupModel, table=True): # type: ignore - pass + grid_squares: Optional[List["GridSquare"]] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) + search_maps: Optional[List["SearchMap"]] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) class NotificationParameter(SQLModel, table=True): # type: ignore @@ -481,6 +495,12 @@ class DataCollection(SQLModel, table=True): # type: ignore movies: List["Movie"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) + motion_correction: Optional[List["MotionCorrection"]] = Relationship( + back_populates="data_collection" + ) + tomogram: Optional[List["Tomogram"]] = Relationship( + back_populates="data_collection" + ) class ProcessingJob(SQLModel, table=True): # type: ignore @@ -582,6 +602,22 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore murfey_ids: List["MurfeyLedger"] = Relationship( back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) + motion_correction: Optional[List["MotionCorrection"]] = Relationship( + back_populates="data_collection" + ) + tomogram: Optional[List["Tomogram"]] = Relationship( + back_populates="auto_proc_program" + ) + ctf: Optional[List["CTF"]] = Relationship(back_populates="auto_proc_program") + particle_picker: Optional[List["ParticlePicker"]] = Relationship( + back_populates="auto_proc_program" + ) + relative_ice_thickness: Optional[List["RelativeIceThickness"]] = Relationship( + back_populates="auto_proc_program" + ) + particle_classification_group: Optional[List["ParticleClassificationGroup"]] = ( + Relationship(back_populates="auto_proc_program") + ) class MurfeyLedger(SQLModel, table=True): # type: ignore @@ -640,6 +676,17 @@ class GridSquare(SQLModel, table=True): # type: ignore foil_holes: List["FoilHole"] = Relationship( back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"} ) + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="grid_squares" + ) class FoilHole(SQLModel, table=True): # type: ignore @@ -665,6 +712,11 @@ class FoilHole(SQLModel, table=True): # type: ignore preprocess_stashes: List[PreprocessStash] = Relationship( back_populates="foil_hole", sa_relationship_kwargs={"cascade": "delete"} ) + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + diameter: Optional[int] = None + quality_indicator: Optional[float] = None class SearchMap(SQLModel, table=True): # type: ignore @@ -697,6 +749,18 @@ class SearchMap(SQLModel, table=True): # type: ignore tilt_series: List["TiltSeries"] = Relationship( back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"} ) + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + scaled_height: Optional[int] = None + scaled_width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="search_maps" + ) + tomogram: Optional[List["Tomogram"]] = Relationship(back_populates="search_map") class Movie(SQLModel, table=True): # type: ignore @@ -710,6 +774,12 @@ class Movie(SQLModel, table=True): # type: ignore murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") + motion_correction: Optional[List["MotionCorrection"]] = Relationship( + back_populates="movie" + ) + tilt_image_alignment: Optional[List["TiltImageAlignment"]] = Relationship( + back_populates="movie" + ) class CtfParameters(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index af5005465..545591043 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -7,85 +7,6 @@ from murfey.util import db -class DataCollectionGroup(db.DataCollectionGroupModel, table=True): # type: ignore - grid_squares: List["GridSquare"] = Relationship( - back_populates="data_collection_group", - sa_relationship_kwargs={"cascade": "delete"}, - ) - search_maps: List["SearchMap"] = Relationship( - back_populates="data_collection_group", - sa_relationship_kwargs={"cascade": "delete"}, - ) - - -class DataCollection(db.DataCollection): - MotionCorrection: List["MotionCorrection"] = Relationship( - back_populates="DataCollection" - ) - Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection") - - -class AutoProcProgram(db.AutoProcProgram): - MotionCorrection: List["MotionCorrection"] = Relationship( - back_populates="DataCollection" - ) - Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram") - CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram") - ParticlePicker: List["ParticlePicker"] = Relationship( - back_populates="AutoProcProgram" - ) - RelativeIceThickness: List["RelativeIceThickness"] = Relationship( - back_populates="AutoProcProgram" - ) - ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( - back_populates="AutoProcProgram" - ) - - -class GridSquare(db.GridSquare): - atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") - scaled_pixel_size: Optional[float] = None - pixel_location_x: Optional[int] = None - pixel_location_y: Optional[int] = None - height: Optional[int] = None - width: Optional[int] = None - angle: Optional[float] = None - quality_indicator: Optional[float] = None - data_collection_group: Optional["DataCollectionGroup"] = Relationship( - back_populates="grid_squares" - ) - - -class FoilHole(db.FoilHole): - scaled_pixel_size: Optional[float] = None - pixel_location_x: Optional[int] = None - pixel_location_y: Optional[int] = None - diameter: Optional[int] = None - quality_indicator: Optional[float] = None - - -class SearchMap(db.SearchMap): - atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") - scaled_pixel_size: Optional[float] = None - pixel_location_x: Optional[int] = None - pixel_location_y: Optional[int] = None - scaled_height: Optional[int] = None - scaled_width: Optional[int] = None - angle: Optional[float] = None - quality_indicator: Optional[float] = None - data_collection_group: Optional["DataCollectionGroup"] = Relationship( - back_populates="search_maps" - ) - Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap") - - -class Movie(db.Movie): - MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") - TiltImageAlignment: List["TiltImageAlignment"] = Relationship( - back_populates="Movie" - ) - - class MotionCorrection(db.SQLModel, table=True): # type: ignore motionCorrectionId: int = Field(primary_key=True, unique=True) dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") @@ -106,19 +27,19 @@ class MotionCorrection(db.SQLModel, table=True): # type: ignore fftCorrectedFullPath: Optional[str] = None comments: Optional[str] = None movieId: Optional[int] = Field(foreign_key="Movie.murfey_id") - AutoProcProgram: Optional["AutoProcProgram"] = Relationship( - back_populates="MotionCorrection" + auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( + back_populates="motion_correction" ) - DataCollection: Optional["DataCollection"] = Relationship( - back_populates="MotionCorrection" + data_collection: Optional["db.DataCollection"] = Relationship( + back_populates="motion_correction" ) - Movie: Optional["Movie"] = Relationship(back_populates="MotionCorrection") - CTF: List["CTF"] = Relationship(back_populates="MotionCorrection") - ParticlePicker: List["ParticlePicker"] = Relationship( - back_populates="MotionCorrection" + movie: Optional["db.Movie"] = Relationship(back_populates="motion_correction") + ctf: List["CTF"] = Relationship(back_populates="motion_correction") + particle_picker: List["ParticlePicker"] = Relationship( + back_populates="motion_correction" ) - RelativeIceThickness: List["RelativeIceThickness"] = Relationship( - back_populates="MotionCorrection" + relative_ice_thickness: List["RelativeIceThickness"] = Relationship( + back_populates="motion_correction" ) @@ -143,8 +64,10 @@ class CTF(db.SQLModel, table=True): # type: ignore ccValue: Optional[float] = None fftTheoreticalFullPath: Optional[str] = None comments: Optional[str] = None - AutoProcProgram: Optional["AutoProcProgram"] = Relationship(back_populates="CTF") - MotionCorrection: Optional["MotionCorrection"] = Relationship(back_populates="CTF") + auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( + back_populates="ctf" + ) + motion_correction: Optional["MotionCorrection"] = Relationship(back_populates="ctf") class ParticlePicker(db.SQLModel, table=True): # type: ignore @@ -157,14 +80,14 @@ class ParticlePicker(db.SQLModel, table=True): # type: ignore particleDiameter: Optional[float] = None numberOfParticles: Optional[int] = None summaryImageFullPath: Optional[str] = None - MotionCorrection: Optional["MotionCorrection"] = Relationship( - back_populates="ParticlePicker" + motion_correction: Optional["MotionCorrection"] = Relationship( + back_populates="particle_picker" ) - AutoProcProgram: Optional["AutoProcProgram"] = Relationship( - back_populates="ParticlePicker" + auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( + back_populates="particle_picker" ) - ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( - back_populates="ParticlePicker" + particle_classification_group: List["ParticleClassificationGroup"] = Relationship( + back_populates="particle_picker" ) @@ -194,16 +117,18 @@ class Tomogram(db.SQLModel, table=True): # type: ignore gridSquareId: Optional[int] = Field(foreign_key="SearchMap.id") pixelLocationX: Optional[int] = None pixelLocationY: Optional[int] = None - AutoProcProgram: Optional["AutoProcProgram"] = Relationship( - back_populates="Tomogram" + auto_program_program: Optional["db.AutoProcProgram"] = Relationship( + back_populates="tomogram" + ) + data_collection: Optional["db.DataCollection"] = Relationship( + back_populates="tomogram" ) - DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") - SearchMap: Optional["SearchMap"] = Relationship(back_populates="Tomogram") - ProcessedTomogram: List["ProcessedTomogram"] = Relationship( - back_populates="Tomogram" + search_map: Optional["db.SearchMap"] = Relationship(back_populates="tomogram") + processed_tomogram: List["ProcessedTomogram"] = Relationship( + back_populates="tomogram" ) - TiltImageAlignment: List["TiltImageAlignment"] = Relationship( - back_populates="Tomogram" + tilt_image_alignment: List["TiltImageAlignment"] = Relationship( + back_populates="tomogram" ) @@ -212,7 +137,7 @@ class ProcessedTomogram(db.SQLModel, table=True): # type: ignore tomogramId: int = Field(foreign_key="Tomogram.tomogramId") filePath: Optional[str] = None processingType: Optional[str] = None - Tomogram: Optional["Tomogram"] = Relationship(back_populates="ProcessedTomogram") + tomogram: Optional["Tomogram"] = Relationship(back_populates="processed_tomogram") class RelativeIceThickness(db.SQLModel, table=True): # type: ignore @@ -228,11 +153,11 @@ class RelativeIceThickness(db.SQLModel, table=True): # type: ignore median: Optional[float] = None q3: Optional[float] = None maximum: Optional[float] = None - AutoProcProgram: Optional["AutoProcProgram"] = Relationship( - back_populates="RelativeIceThickness" + auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( + back_populates="relative_ice_thickness" ) - MotionCorrection: Optional["MotionCorrection"] = Relationship( - back_populates="RelativeIceThickness" + motion_correction: Optional["MotionCorrection"] = Relationship( + back_populates="relative_ice_thickness" ) @@ -248,8 +173,8 @@ class TiltImageAlignment(db.SQLModel, table=True): # type: ignore refinedTiltAngle: Optional[float] = None refinedTiltAxis: Optional[float] = None residualError: Optional[float] = None - Movie: Optional["Movie"] = Relationship(back_populates="TiltImageAlignment") - Tomogram: Optional["Tomogram"] = Relationship(back_populates="TiltImageAlignment") + movie: Optional["db.Movie"] = Relationship(back_populates="tilt_image_alignment") + tomogram: Optional["Tomogram"] = Relationship(back_populates="tilt_image_alignment") class ParticleClassificationGroup(db.SQLModel, table=True): # type: ignore @@ -264,14 +189,14 @@ class ParticleClassificationGroup(db.SQLModel, table=True): # type: ignore numberOfClassesPerBatch: Optional[int] = None symmetry: Optional[str] = None binnedPixelSize: Optional[float] = None - ParticlePicker: Optional["ParticlePicker"] = Relationship( - back_populates="ParticleClassificationGroup" + particle_picker: Optional["ParticlePicker"] = Relationship( + back_populates="particle_classification_group" ) - AutoProcProgram: Optional["AutoProcProgram"] = Relationship( - back_populates="ParticleClassificationGroup" + auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( + back_populates="particle_classification_group" ) - ParticleClassification: List["ParticleClassification"] = Relationship( - back_populates="ParticleClassificationGroup" + particle_classification: List["ParticleClassification"] = Relationship( + back_populates="particle_classification_group" ) @@ -294,14 +219,14 @@ class ParticleClassification(db.SQLModel, table=True): # type: ignore bFactorFitQuadratic: Optional[float] = None angularEfficiency: Optional[float] = None suggestedTilt: Optional[float] = None - CryoemInitialModel: List["CryoemInitialModel"] = Relationship( - back_populates="ParticleClassification" + cryoem_initial_model: List["CryoemInitialModel"] = Relationship( + back_populates="particle_classification" ) - ParticleClassificationGroup: Optional["ParticleClassificationGroup"] = Relationship( - back_populates="ParticleClassification" + particle_classification_group: Optional["ParticleClassificationGroup"] = ( + Relationship(back_populates="particle_classification") ) - BFactorFit: List["BFactorFit"] = Relationship( - back_populates="ParticleClassification" + bfactor_fit: List["BFactorFit"] = Relationship( + back_populates="particle_classification" ) @@ -313,8 +238,8 @@ class BFactorFit(db.SQLModel, table=True): # type: ignore resolution: Optional[float] = None numberOfParticles: Optional[int] = None particleBatchSize: Optional[int] = None - ParticleClassification: Optional["ParticleClassification"] = Relationship( - back_populates="BFactorFit" + particle_classification: Optional["ParticleClassification"] = Relationship( + back_populates="bfactor_fit" ) @@ -325,8 +250,8 @@ class CryoemInitialModel(db.SQLModel, table=True): # type: ignore ) resolution: Optional[float] = None numberOfParticles: Optional[int] = None - ParticleClassification: List["ParticleClassification"] = Relationship( - back_populates="CryoemInitialModel" + particle_classification: List["ParticleClassification"] = Relationship( + back_populates="cryoem_initial_model" ) From 74622612b1569721ae3ef29cfff64292c5fc71ab Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 10:32:02 +0000 Subject: [PATCH 23/40] Test imported wrong thing --- tests/util/test_processing_db.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/util/test_processing_db.py b/tests/util/test_processing_db.py index 528e4159b..82ca2337d 100644 --- a/tests/util/test_processing_db.py +++ b/tests/util/test_processing_db.py @@ -9,7 +9,6 @@ CTF, BFactorFit, CryoemInitialModel, - DataCollectionGroup, MotionCorrection, ParticleClassification, ParticleClassificationGroup, @@ -47,7 +46,6 @@ def test_processing_tables_exist(murfey_db_url): with sessionmaker( bind=connection, expire_on_commit=False, class_=Session ) as murfey_db_session: - assert murfey_db_session.exec(select(DataCollectionGroup)).all() == [] assert murfey_db_session.exec(select(MotionCorrection)).all() == [] assert murfey_db_session.exec(select(CTF)).all() == [] assert murfey_db_session.exec(select(ParticlePicker)).all() == [] @@ -60,9 +58,10 @@ def test_processing_tables_exist(murfey_db_url): assert murfey_db_session.exec(select(BFactorFit)).all() == [] assert murfey_db_session.exec(select(CryoemInitialModel)).all() == [] - from murfey.util.db import ClientEnvironment + from murfey.util.db import ClientEnvironment, DataCollectionGroup assert murfey_db_session.exec(select(ClientEnvironment)).all() == [] + assert murfey_db_session.exec(select(DataCollectionGroup)).all() == [] connection.close() metadata = MetaData() From 49bb0640c074a34303e261c023904165a97a19ac Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 10:43:08 +0000 Subject: [PATCH 24/40] foreign keys needed --- src/murfey/util/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 1beb9e7b2..8dc00a35a 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -486,6 +486,8 @@ class DataCollection(SQLModel, table=True): # type: ignore id: int = Field(primary_key=True, unique=True) tag: str = Field(primary_key=True) dcg_id: int = Field(foreign_key="datacollectiongroup.id") + mc_id: Optional[int] = Field(foreign_key="motioncorrection.motionCorrectionId") + tomogram_id: Optional[int] = Field(foreign_key="tomogram.tomogramId") data_collection_group: Optional[DataCollectionGroup] = Relationship( back_populates="data_collections" ) From c14bc21b7c88fdc446ca2e05f84dbc647cc44d0a Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 10:51:04 +0000 Subject: [PATCH 25/40] Add relations --- src/murfey/util/db.py | 2 ++ src/murfey/util/processing_db.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 8dc00a35a..b03231b78 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -598,6 +598,8 @@ class TomographyProcessingParameters(SQLModel, table=True): # type: ignore class AutoProcProgram(SQLModel, table=True): # type: ignore id: int = Field(primary_key=True, unique=True) pj_id: int = Field(foreign_key="processingjob.id") + mc_id: Optional[int] = Field(foreign_key="motioncorrection.motionCorrectionId") + ctf_id: Optional[int] = Field(foreign_key="ctf.ctfId") processing_job: Optional[ProcessingJob] = Relationship( back_populates="auto_proc_programs" ) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 545591043..f6af3d8c7 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -9,8 +9,8 @@ class MotionCorrection(db.SQLModel, table=True): # type: ignore motionCorrectionId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + dataCollectionId: Optional[int] = Field(foreign_key="db.DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="db.AutoProgProgram.id") imageNumber: Optional[int] = None firstFrame: Optional[int] = None lastFrame: Optional[int] = None @@ -48,7 +48,7 @@ class CTF(db.SQLModel, table=True): # type: ignore motionCorrectionId: Optional[int] = Field( foreign_key="MotionCorrection.motionCorrectionId" ) - autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + autoProcProgramId: Optional[int] = Field(foreign_key="db.AutoProcProgram.id") boxSizeX: Optional[float] = None boxSizeY: Optional[float] = None minResolution: Optional[float] = None @@ -72,7 +72,7 @@ class CTF(db.SQLModel, table=True): # type: ignore class ParticlePicker(db.SQLModel, table=True): # type: ignore particlePickerId: int = Field(primary_key=True, unique=True) - programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + programId: Optional[int] = Field(foreign_key="db.AutoProcProgram.autoProcProgramId") firstMotionCorrectionId: Optional[int] = Field( foreign_key="MotionCorrection.motionCorrectionId" ) @@ -93,8 +93,8 @@ class ParticlePicker(db.SQLModel, table=True): # type: ignore class Tomogram(db.SQLModel, table=True): # type: ignore tomogramId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + dataCollectionId: Optional[int] = Field(foreign_key="db.DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="db.AutoProcProgram.id") volumeFile: Optional[str] = None stackFile: Optional[str] = None sizeX: Optional[int] = None @@ -146,7 +146,7 @@ class RelativeIceThickness(db.SQLModel, table=True): # type: ignore foreign_key="MotionCorrection.motionCorrectionId" ) autoProcProgramId: Optional[int] = Field( - foreign_key="AutoProcProgram.autoProcProgramId" + foreign_key="db.AutoProcProgram.autoProcProgramId" ) minimum: Optional[float] = None q1: Optional[float] = None @@ -162,7 +162,7 @@ class RelativeIceThickness(db.SQLModel, table=True): # type: ignore class TiltImageAlignment(db.SQLModel, table=True): # type: ignore - movieId: int = Field(foreign_key="Movie.murfey_id", primary_key=True) + movieId: int = Field(foreign_key="db.Movie.murfey_id", primary_key=True) tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) defocusU: Optional[float] = None defocusV: Optional[float] = None @@ -182,7 +182,7 @@ class ParticleClassificationGroup(db.SQLModel, table=True): # type: ignore particlePickerId: Optional[int] = Field( foreign_key="ParticlePicker.particlePickerId" ) - programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + programId: Optional[int] = Field(foreign_key="db.AutoProcProgram.autoProcProgramId") type: Optional[str] = Enum("2D", "3D") batchNumber: Optional[int] = None numberOfParticlesPerBatch: Optional[int] = None From 6913633ea7ec9c2232645c1da88bdbdf8844b7d8 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:06:04 +0000 Subject: [PATCH 26/40] Maybe that was wrong --- src/murfey/util/processing_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index f6af3d8c7..f8eb6b26b 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -9,8 +9,8 @@ class MotionCorrection(db.SQLModel, table=True): # type: ignore motionCorrectionId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="db.DataCollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="db.AutoProgProgram.id") + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProgProgram.id") imageNumber: Optional[int] = None firstFrame: Optional[int] = None lastFrame: Optional[int] = None From 2aa9f98d64e78dc07c1d04227023c3f44d5df52d Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:16:33 +0000 Subject: [PATCH 27/40] That was daft --- src/murfey/util/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index b03231b78..5698749b8 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -607,7 +607,7 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) motion_correction: Optional[List["MotionCorrection"]] = Relationship( - back_populates="data_collection" + back_populates="motion_correction" ) tomogram: Optional[List["Tomogram"]] = Relationship( back_populates="auto_proc_program" From c56ce1ea54ab1ab58f88b6d89fa4f9252622fe52 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:24:00 +0000 Subject: [PATCH 28/40] That was also daft --- src/murfey/util/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 5698749b8..c69ebbc8b 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -607,7 +607,7 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) motion_correction: Optional[List["MotionCorrection"]] = Relationship( - back_populates="motion_correction" + back_populates="auto_proc_program" ) tomogram: Optional[List["Tomogram"]] = Relationship( back_populates="auto_proc_program" From ee715f5c5f175c4bf3345343da3d58004be9c172 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:32:09 +0000 Subject: [PATCH 29/40] More foreign keys --- src/murfey/util/db.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index c69ebbc8b..442eb9bf6 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -598,14 +598,24 @@ class TomographyProcessingParameters(SQLModel, table=True): # type: ignore class AutoProcProgram(SQLModel, table=True): # type: ignore id: int = Field(primary_key=True, unique=True) pj_id: int = Field(foreign_key="processingjob.id") - mc_id: Optional[int] = Field(foreign_key="motioncorrection.motionCorrectionId") - ctf_id: Optional[int] = Field(foreign_key="ctf.ctfId") processing_job: Optional[ProcessingJob] = Relationship( back_populates="auto_proc_programs" ) murfey_ids: List["MurfeyLedger"] = Relationship( back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) + mc_id: Optional[int] = Field(foreign_key="motioncorrection.motionCorrectionId") + ctf_id: Optional[int] = Field(foreign_key="ctf.ctfId") + tomogram_id: Optional[int] = Field(foreign_key="tomogram.tomogramId") + particle_picker_id: Optional[int] = Field( + foreign_key="ParticlePicker.particlePickerId" + ) + relative_ice_thickness_id: Optional[int] = Field( + foreign_key="RelativeIceThickness.relativeIceThicknessId" + ) + particle_classification_group_id: Optional[int] = Field( + foreign_key="ParticleClassificationGroup.particleClassificationGroupId" + ) motion_correction: Optional[List["MotionCorrection"]] = Relationship( back_populates="auto_proc_program" ) From 08e3c9aa183e70538385c90fbd480878d50e1133 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:42:35 +0000 Subject: [PATCH 30/40] Casings and typo --- src/murfey/util/db.py | 6 ++--- src/murfey/util/processing_db.py | 42 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 442eb9bf6..1f7c9b167 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -608,13 +608,13 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore ctf_id: Optional[int] = Field(foreign_key="ctf.ctfId") tomogram_id: Optional[int] = Field(foreign_key="tomogram.tomogramId") particle_picker_id: Optional[int] = Field( - foreign_key="ParticlePicker.particlePickerId" + foreign_key="particlepicker.particlePickerId" ) relative_ice_thickness_id: Optional[int] = Field( - foreign_key="RelativeIceThickness.relativeIceThicknessId" + foreign_key="relativeicethickness.relativeIceThicknessId" ) particle_classification_group_id: Optional[int] = Field( - foreign_key="ParticleClassificationGroup.particleClassificationGroupId" + foreign_key="particleclassificationgroup.particleClassificationGroupId" ) motion_correction: Optional[List["MotionCorrection"]] = Relationship( back_populates="auto_proc_program" diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index f8eb6b26b..a3be68bca 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -9,8 +9,8 @@ class MotionCorrection(db.SQLModel, table=True): # type: ignore motionCorrectionId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="AutoProgProgram.id") + dataCollectionId: Optional[int] = Field(foreign_key="datacollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") imageNumber: Optional[int] = None firstFrame: Optional[int] = None lastFrame: Optional[int] = None @@ -26,7 +26,7 @@ class MotionCorrection(db.SQLModel, table=True): # type: ignore fftFullPath: Optional[str] = None fftCorrectedFullPath: Optional[str] = None comments: Optional[str] = None - movieId: Optional[int] = Field(foreign_key="Movie.murfey_id") + movieId: Optional[int] = Field(foreign_key="movie.murfey_id") auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( back_populates="motion_correction" ) @@ -46,9 +46,9 @@ class MotionCorrection(db.SQLModel, table=True): # type: ignore class CTF(db.SQLModel, table=True): # type: ignore ctfId: int = Field(primary_key=True, unique=True) motionCorrectionId: Optional[int] = Field( - foreign_key="MotionCorrection.motionCorrectionId" + foreign_key="motioncorrection.motionCorrectionId" ) - autoProcProgramId: Optional[int] = Field(foreign_key="db.AutoProcProgram.id") + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") boxSizeX: Optional[float] = None boxSizeY: Optional[float] = None minResolution: Optional[float] = None @@ -72,9 +72,9 @@ class CTF(db.SQLModel, table=True): # type: ignore class ParticlePicker(db.SQLModel, table=True): # type: ignore particlePickerId: int = Field(primary_key=True, unique=True) - programId: Optional[int] = Field(foreign_key="db.AutoProcProgram.autoProcProgramId") + programId: Optional[int] = Field(foreign_key="autoprocprogram.autoProcProgramId") firstMotionCorrectionId: Optional[int] = Field( - foreign_key="MotionCorrection.motionCorrectionId" + foreign_key="motioncorrection.motionCorrectionId" ) particlePickingTemplate: Optional[str] = None particleDiameter: Optional[float] = None @@ -93,8 +93,8 @@ class ParticlePicker(db.SQLModel, table=True): # type: ignore class Tomogram(db.SQLModel, table=True): # type: ignore tomogramId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="db.DataCollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="db.AutoProcProgram.id") + dataCollectionId: Optional[int] = Field(foreign_key="datacollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") volumeFile: Optional[str] = None stackFile: Optional[str] = None sizeX: Optional[int] = None @@ -114,10 +114,10 @@ class Tomogram(db.SQLModel, table=True): # type: ignore projXZ: Optional[str] = None recordTimeStamp: Optional[datetime.datetime] = None globalAlignmentQuality: Optional[float] = None - gridSquareId: Optional[int] = Field(foreign_key="SearchMap.id") + gridSquareId: Optional[int] = Field(foreign_key="searchmap.id") pixelLocationX: Optional[int] = None pixelLocationY: Optional[int] = None - auto_program_program: Optional["db.AutoProcProgram"] = Relationship( + auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( back_populates="tomogram" ) data_collection: Optional["db.DataCollection"] = Relationship( @@ -134,7 +134,7 @@ class Tomogram(db.SQLModel, table=True): # type: ignore class ProcessedTomogram(db.SQLModel, table=True): # type: ignore processedTomogramId: int = Field(primary_key=True, unique=True) - tomogramId: int = Field(foreign_key="Tomogram.tomogramId") + tomogramId: int = Field(foreign_key="tomogram.tomogramId") filePath: Optional[str] = None processingType: Optional[str] = None tomogram: Optional["Tomogram"] = Relationship(back_populates="processed_tomogram") @@ -143,10 +143,10 @@ class ProcessedTomogram(db.SQLModel, table=True): # type: ignore class RelativeIceThickness(db.SQLModel, table=True): # type: ignore relativeIceThicknessId: int = Field(primary_key=True, unique=True) motionCorrectionId: Optional[int] = Field( - foreign_key="MotionCorrection.motionCorrectionId" + foreign_key="motioncorrection.motionCorrectionId" ) autoProcProgramId: Optional[int] = Field( - foreign_key="db.AutoProcProgram.autoProcProgramId" + foreign_key="autoprocprogram.autoProcProgramId" ) minimum: Optional[float] = None q1: Optional[float] = None @@ -162,8 +162,8 @@ class RelativeIceThickness(db.SQLModel, table=True): # type: ignore class TiltImageAlignment(db.SQLModel, table=True): # type: ignore - movieId: int = Field(foreign_key="db.Movie.murfey_id", primary_key=True) - tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) + movieId: int = Field(foreign_key="movie.murfey_id", primary_key=True) + tomogramId: int = Field(foreign_key="tomogram.tomogramId", primary_key=True) defocusU: Optional[float] = None defocusV: Optional[float] = None psdFile: Optional[str] = None @@ -180,9 +180,9 @@ class TiltImageAlignment(db.SQLModel, table=True): # type: ignore class ParticleClassificationGroup(db.SQLModel, table=True): # type: ignore particleClassificationGroupId: int = Field(primary_key=True, unique=True) particlePickerId: Optional[int] = Field( - foreign_key="ParticlePicker.particlePickerId" + foreign_key="particlepicker.particlePickerId" ) - programId: Optional[int] = Field(foreign_key="db.AutoProcProgram.autoProcProgramId") + programId: Optional[int] = Field(foreign_key="autoprocprogram.autoProcProgramId") type: Optional[str] = Enum("2D", "3D") batchNumber: Optional[int] = None numberOfParticlesPerBatch: Optional[int] = None @@ -210,7 +210,7 @@ class ParticleClassification(db.SQLModel, table=True): # type: ignore estimatedResolution: Optional[float] = None overallFourierCompleteness: Optional[float] = None particleClassificationGroupId: Optional[int] = Field( - foreign_key="ParticleClassificationGroup.particleClassificationGroupId" + foreign_key="particleclassificationgroup.particleClassificationGroupId" ) classDistribution: Optional[float] = None selected: Optional[int] = None @@ -233,7 +233,7 @@ class ParticleClassification(db.SQLModel, table=True): # type: ignore class BFactorFit(db.SQLModel, table=True): # type: ignore bFactorFitId: int = Field(primary_key=True, unique=True) particleClassificationId: int = Field( - foreign_key="ParticleClassification.particleClassificationId" + foreign_key="particleclassification.particleClassificationId" ) resolution: Optional[float] = None numberOfParticles: Optional[int] = None @@ -246,7 +246,7 @@ class BFactorFit(db.SQLModel, table=True): # type: ignore class CryoemInitialModel(db.SQLModel, table=True): # type: ignore cryoemInitialModelId: int = Field(primary_key=True, unique=True) particleClassificationId: int = Field( - foreign_key="ParticleClassification.particleClassificationId" + foreign_key="particleclassification.particleClassificationId" ) resolution: Optional[float] = None numberOfParticles: Optional[int] = None From b6adae72322e18d165a796772c946b7a16f0f410 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:48:34 +0000 Subject: [PATCH 31/40] Try reverting some of the keys --- src/murfey/util/db.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 1f7c9b167..49ebb1ed1 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -486,8 +486,6 @@ class DataCollection(SQLModel, table=True): # type: ignore id: int = Field(primary_key=True, unique=True) tag: str = Field(primary_key=True) dcg_id: int = Field(foreign_key="datacollectiongroup.id") - mc_id: Optional[int] = Field(foreign_key="motioncorrection.motionCorrectionId") - tomogram_id: Optional[int] = Field(foreign_key="tomogram.tomogramId") data_collection_group: Optional[DataCollectionGroup] = Relationship( back_populates="data_collections" ) @@ -604,18 +602,6 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore murfey_ids: List["MurfeyLedger"] = Relationship( back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) - mc_id: Optional[int] = Field(foreign_key="motioncorrection.motionCorrectionId") - ctf_id: Optional[int] = Field(foreign_key="ctf.ctfId") - tomogram_id: Optional[int] = Field(foreign_key="tomogram.tomogramId") - particle_picker_id: Optional[int] = Field( - foreign_key="particlepicker.particlePickerId" - ) - relative_ice_thickness_id: Optional[int] = Field( - foreign_key="relativeicethickness.relativeIceThicknessId" - ) - particle_classification_group_id: Optional[int] = Field( - foreign_key="particleclassificationgroup.particleClassificationGroupId" - ) motion_correction: Optional[List["MotionCorrection"]] = Relationship( back_populates="auto_proc_program" ) From fda1f748e769c3d5fb767fb29ac1f983ffe2a81a Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:50:01 +0000 Subject: [PATCH 32/40] Wrong keys --- src/murfey/util/processing_db.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index a3be68bca..5a5202748 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -72,7 +72,7 @@ class CTF(db.SQLModel, table=True): # type: ignore class ParticlePicker(db.SQLModel, table=True): # type: ignore particlePickerId: int = Field(primary_key=True, unique=True) - programId: Optional[int] = Field(foreign_key="autoprocprogram.autoProcProgramId") + programId: Optional[int] = Field(foreign_key="autoprocprogram.id") firstMotionCorrectionId: Optional[int] = Field( foreign_key="motioncorrection.motionCorrectionId" ) @@ -145,9 +145,7 @@ class RelativeIceThickness(db.SQLModel, table=True): # type: ignore motionCorrectionId: Optional[int] = Field( foreign_key="motioncorrection.motionCorrectionId" ) - autoProcProgramId: Optional[int] = Field( - foreign_key="autoprocprogram.autoProcProgramId" - ) + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") minimum: Optional[float] = None q1: Optional[float] = None median: Optional[float] = None @@ -182,7 +180,7 @@ class ParticleClassificationGroup(db.SQLModel, table=True): # type: ignore particlePickerId: Optional[int] = Field( foreign_key="particlepicker.particlePickerId" ) - programId: Optional[int] = Field(foreign_key="autoprocprogram.autoProcProgramId") + programId: Optional[int] = Field(foreign_key="autoprocprogram.id") type: Optional[str] = Enum("2D", "3D") batchNumber: Optional[int] = None numberOfParticlesPerBatch: Optional[int] = None From 55388f49bc8b4feaa35b2f8b7309206c6d3d3ba7 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 14:55:38 +0000 Subject: [PATCH 33/40] missed bracket layer? --- tests/util/test_processing_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/util/test_processing_db.py b/tests/util/test_processing_db.py index 82ca2337d..c39592083 100644 --- a/tests/util/test_processing_db.py +++ b/tests/util/test_processing_db.py @@ -45,7 +45,7 @@ def test_processing_tables_exist(murfey_db_url): with sessionmaker( bind=connection, expire_on_commit=False, class_=Session - ) as murfey_db_session: + )() as murfey_db_session: assert murfey_db_session.exec(select(MotionCorrection)).all() == [] assert murfey_db_session.exec(select(CTF)).all() == [] assert murfey_db_session.exec(select(ParticlePicker)).all() == [] From 7b340983556bc81e90207143faa8c532c742458b Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 15:04:37 +0000 Subject: [PATCH 34/40] Try without the metadata bit --- tests/util/test_processing_db.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/util/test_processing_db.py b/tests/util/test_processing_db.py index c39592083..9dd2ef9fe 100644 --- a/tests/util/test_processing_db.py +++ b/tests/util/test_processing_db.py @@ -1,7 +1,7 @@ import os import pytest -from sqlalchemy import MetaData, create_engine +from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlmodel import Session, select @@ -64,7 +64,3 @@ def test_processing_tables_exist(murfey_db_url): assert murfey_db_session.exec(select(DataCollectionGroup)).all() == [] connection.close() - metadata = MetaData() - metadata.create_all(engine) - metadata.reflect(engine) - metadata.drop_all(engine) From c3a3e185cc1d42e58a04c58d617dc1c4dc400c1a Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 20 Feb 2026 15:51:09 +0000 Subject: [PATCH 35/40] No need for mapper registry --- src/murfey/util/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 34d7e4cbf..eec5fc4cc 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -23,7 +23,6 @@ """ GENERAL """ -mapper_registry = sqlalchemy.orm.registry() class MurfeyUser(SQLModel, table=True): # type: ignore From 7fe54e52da6f5a0ed1ef54f619a138ffefdf2d5b Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Thu, 26 Feb 2026 11:21:23 +0000 Subject: [PATCH 36/40] combine schema files to avoid missing import errors from sqlalchemy session --- src/murfey/util/db.py | 259 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 246 insertions(+), 13 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index eec5fc4cc..c1ed79681 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -4,21 +4,10 @@ """ from datetime import datetime -from typing import TYPE_CHECKING, List, Optional +from typing import List, Optional import sqlalchemy -from sqlmodel import Field, Relationship, SQLModel, create_engine - -if TYPE_CHECKING: - from murfey.util.processing_db import ( - CTF, - MotionCorrection, - ParticleClassificationGroup, - ParticlePicker, - RelativeIceThickness, - TiltImageAlignment, - Tomogram, - ) +from sqlmodel import Enum, Field, Relationship, SQLModel, create_engine """ GENERAL @@ -968,6 +957,250 @@ class BFactors(SQLModel, table=True): # type: ignore resolution: float +class MotionCorrection(SQLModel, table=True): # type: ignore + motionCorrectionId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="datacollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") + imageNumber: Optional[int] = None + firstFrame: Optional[int] = None + lastFrame: Optional[int] = None + dosePerFrame: Optional[float] = None + doseWeight: Optional[float] = None + totalMotion: Optional[float] = None + averageMotionPerFrame: Optional[float] = None + driftPlotFullPath: Optional[str] = None + micrographFullPath: Optional[str] = None + micrographSnapshotFullPath: Optional[str] = None + patchesUsedX: Optional[int] = None + patchesUsedY: Optional[int] = None + fftFullPath: Optional[str] = None + fftCorrectedFullPath: Optional[str] = None + comments: Optional[str] = None + movieId: Optional[int] = Field(foreign_key="movie.murfey_id") + auto_proc_program: Optional["AutoProcProgram"] = Relationship( + back_populates="motion_correction" + ) + data_collection: Optional["DataCollection"] = Relationship( + back_populates="motion_correction" + ) + movie: Optional["Movie"] = Relationship(back_populates="motion_correction") + ctf: List["CTF"] = Relationship(back_populates="motion_correction") + particle_picker: List["ParticlePicker"] = Relationship( + back_populates="motion_correction" + ) + relative_ice_thickness: List["RelativeIceThickness"] = Relationship( + back_populates="motion_correction" + ) + + +class CTF(SQLModel, table=True): # type: ignore + ctfId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="motioncorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") + boxSizeX: Optional[float] = None + boxSizeY: Optional[float] = None + minResolution: Optional[float] = None + maxResolution: Optional[float] = None + minDefocus: Optional[float] = None + maxDefocus: Optional[float] = None + defocusStepSize: Optional[float] = None + astigmatism: Optional[float] = None + astigmatismAngle: Optional[float] = None + estimatedResolution: Optional[float] = None + estimatedDefocus: Optional[float] = None + amplitudeContrast: Optional[float] = None + ccValue: Optional[float] = None + fftTheoreticalFullPath: Optional[str] = None + comments: Optional[str] = None + auto_proc_program: Optional["AutoProcProgram"] = Relationship(back_populates="ctf") + motion_correction: Optional["MotionCorrection"] = Relationship(back_populates="ctf") + + +class ParticlePicker(SQLModel, table=True): # type: ignore + particlePickerId: int = Field(primary_key=True, unique=True) + programId: Optional[int] = Field(foreign_key="autoprocprogram.id") + firstMotionCorrectionId: Optional[int] = Field( + foreign_key="motioncorrection.motionCorrectionId" + ) + particlePickingTemplate: Optional[str] = None + particleDiameter: Optional[float] = None + numberOfParticles: Optional[int] = None + summaryImageFullPath: Optional[str] = None + motion_correction: Optional["MotionCorrection"] = Relationship( + back_populates="particle_picker" + ) + auto_proc_program: Optional["AutoProcProgram"] = Relationship( + back_populates="particle_picker" + ) + particle_classification_group: List["ParticleClassificationGroup"] = Relationship( + back_populates="particle_picker" + ) + + +class Tomogram(SQLModel, table=True): # type: ignore + tomogramId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="datacollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") + volumeFile: Optional[str] = None + stackFile: Optional[str] = None + sizeX: Optional[int] = None + sizeY: Optional[int] = None + sizeZ: Optional[int] = None + pixelSpacing: Optional[float] = None + residualErrorMean: Optional[float] = None + residualErrorSD: Optional[float] = None + xAxisCorrection: Optional[float] = None + tiltAngleOffset: Optional[float] = None + zShift: Optional[float] = None + fileDirectory: Optional[str] = None + centralSliceImage: Optional[str] = None + tomogramMovie: Optional[str] = None + xyShiftPlot: Optional[str] = None + projXY: Optional[str] = None + projXZ: Optional[str] = None + recordTimeStamp: Optional[datetime] = None + globalAlignmentQuality: Optional[float] = None + gridSquareId: Optional[int] = Field(foreign_key="searchmap.id") + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + auto_proc_program: Optional["AutoProcProgram"] = Relationship( + back_populates="tomogram" + ) + data_collection: Optional["DataCollection"] = Relationship( + back_populates="tomogram" + ) + search_map: Optional["SearchMap"] = Relationship(back_populates="tomogram") + processed_tomogram: List["ProcessedTomogram"] = Relationship( + back_populates="tomogram" + ) + tilt_image_alignment: List["TiltImageAlignment"] = Relationship( + back_populates="tomogram" + ) + + +class ProcessedTomogram(SQLModel, table=True): # type: ignore + processedTomogramId: int = Field(primary_key=True, unique=True) + tomogramId: int = Field(foreign_key="tomogram.tomogramId") + filePath: Optional[str] = None + processingType: Optional[str] = None + tomogram: Optional["Tomogram"] = Relationship(back_populates="processed_tomogram") + + +class RelativeIceThickness(SQLModel, table=True): # type: ignore + relativeIceThicknessId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="motioncorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") + minimum: Optional[float] = None + q1: Optional[float] = None + median: Optional[float] = None + q3: Optional[float] = None + maximum: Optional[float] = None + auto_proc_program: Optional["AutoProcProgram"] = Relationship( + back_populates="relative_ice_thickness" + ) + motion_correction: Optional["MotionCorrection"] = Relationship( + back_populates="relative_ice_thickness" + ) + + +class TiltImageAlignment(SQLModel, table=True): # type: ignore + movieId: int = Field(foreign_key="movie.murfey_id", primary_key=True) + tomogramId: int = Field(foreign_key="tomogram.tomogramId", primary_key=True) + defocusU: Optional[float] = None + defocusV: Optional[float] = None + psdFile: Optional[str] = None + resolution: Optional[float] = None + fitQuality: Optional[float] = None + refinedMagnification: Optional[float] = None + refinedTiltAngle: Optional[float] = None + refinedTiltAxis: Optional[float] = None + residualError: Optional[float] = None + movie: Optional["Movie"] = Relationship(back_populates="tilt_image_alignment") + tomogram: Optional["Tomogram"] = Relationship(back_populates="tilt_image_alignment") + + +class ParticleClassificationGroup(SQLModel, table=True): # type: ignore + particleClassificationGroupId: int = Field(primary_key=True, unique=True) + particlePickerId: Optional[int] = Field( + foreign_key="particlepicker.particlePickerId" + ) + programId: Optional[int] = Field(foreign_key="autoprocprogram.id") + type: Optional[str] = Enum("2D", "3D") + batchNumber: Optional[int] = None + numberOfParticlesPerBatch: Optional[int] = None + numberOfClassesPerBatch: Optional[int] = None + symmetry: Optional[str] = None + binnedPixelSize: Optional[float] = None + particle_picker: Optional["ParticlePicker"] = Relationship( + back_populates="particle_classification_group" + ) + auto_proc_program: Optional["AutoProcProgram"] = Relationship( + back_populates="particle_classification_group" + ) + particle_classification: List["ParticleClassification"] = Relationship( + back_populates="particle_classification_group" + ) + + +class ParticleClassification(SQLModel, table=True): # type: ignore + particleClassificationId: int = Field(primary_key=True, unique=True) + classNumber: Optional[int] = None + classImageFullPath: Optional[str] = None + particlesPerClass: Optional[int] = None + rotationAccuracy: Optional[float] = None + translationAccuracy: Optional[float] = None + estimatedResolution: Optional[float] = None + overallFourierCompleteness: Optional[float] = None + particleClassificationGroupId: Optional[int] = Field( + foreign_key="particleclassificationgroup.particleClassificationGroupId" + ) + classDistribution: Optional[float] = None + selected: Optional[int] = None + bFactorFitIntercept: Optional[float] = None + bFactorFitLinear: Optional[float] = None + bFactorFitQuadratic: Optional[float] = None + angularEfficiency: Optional[float] = None + suggestedTilt: Optional[float] = None + cryoem_initial_model: List["CryoemInitialModel"] = Relationship( + back_populates="particle_classification" + ) + particle_classification_group: Optional["ParticleClassificationGroup"] = ( + Relationship(back_populates="particle_classification") + ) + bfactor_fit: List["BFactorFit"] = Relationship( + back_populates="particle_classification" + ) + + +class BFactorFit(SQLModel, table=True): # type: ignore + bFactorFitId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="particleclassification.particleClassificationId" + ) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + particleBatchSize: Optional[int] = None + particle_classification: Optional["ParticleClassification"] = Relationship( + back_populates="bfactor_fit" + ) + + +class CryoemInitialModel(SQLModel, table=True): # type: ignore + cryoemInitialModelId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="particleclassification.particleClassificationId" + ) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + particle_classification: List["ParticleClassification"] = Relationship( + back_populates="cryoem_initial_model" + ) + + """ FUNCTIONS """ From 462d439a09da7e234075b0ed4ce9388b30b09974 Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Thu, 26 Feb 2026 11:34:22 +0000 Subject: [PATCH 37/40] small changes to try and help us move to numpy 2 --- src/murfey/server/feedback.py | 4 ++-- src/murfey/workflows/tomo/picking.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 72435d27f..f58529da0 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -1819,8 +1819,8 @@ def _save_bfactor(message: dict, _db): "ispyb_command": "insert_particle_classification" }, "program_id": message["program_id"], - "bfactor_fit_intercept": str(bfactor_fitting[1]), - "bfactor_fit_linear": str(bfactor_fitting[0]), + "bfactor_fit_intercept": str(float(bfactor_fitting[1])), + "bfactor_fit_linear": str(float(bfactor_fitting[0])), }, new_connection=True, ) diff --git a/src/murfey/workflows/tomo/picking.py b/src/murfey/workflows/tomo/picking.py index a7432df71..8eda2ab4e 100644 --- a/src/murfey/workflows/tomo/picking.py +++ b/src/murfey/workflows/tomo/picking.py @@ -102,7 +102,7 @@ def _register_picked_tomogram_use_diameter(message: dict, murfey_db: Session): picking_db = murfey_db.exec( select(ParticleSizes.particle_size).where(ParticleSizes.pj_id == pj_id) ).all() - particle_diameter = np.quantile(list(picking_db), 0.75) + particle_diameter = float(np.quantile(list(picking_db), 0.75)) tomo_params.particle_diameter = particle_diameter murfey_db.add(tomo_params) murfey_db.commit() From f615d4c47061d482daf64acc3f7f75bd3c988371 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 26 Feb 2026 15:09:31 +0000 Subject: [PATCH 38/40] Add ice ring density field --- src/murfey/util/db.py | 1 + src/murfey/util/processing_db.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index c1ed79681..d2f41e9ea 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -1013,6 +1013,7 @@ class CTF(SQLModel, table=True): # type: ignore amplitudeContrast: Optional[float] = None ccValue: Optional[float] = None fftTheoreticalFullPath: Optional[str] = None + iceRingDensity: Optional[float] = None comments: Optional[str] = None auto_proc_program: Optional["AutoProcProgram"] = Relationship(back_populates="ctf") motion_correction: Optional["MotionCorrection"] = Relationship(back_populates="ctf") diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 5a5202748..24c2d33b8 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -63,6 +63,7 @@ class CTF(db.SQLModel, table=True): # type: ignore amplitudeContrast: Optional[float] = None ccValue: Optional[float] = None fftTheoreticalFullPath: Optional[str] = None + iceRingDensity: Optional[float] = None comments: Optional[str] = None auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( back_populates="ctf" From 44249aec51ba44ea20da99b79bdc45d8786f28d1 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 26 Feb 2026 15:17:00 +0000 Subject: [PATCH 39/40] REmove processing db from cli --- src/murfey/cli/create_db.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/murfey/cli/create_db.py b/src/murfey/cli/create_db.py index e1add0683..7af3e026f 100644 --- a/src/murfey/cli/create_db.py +++ b/src/murfey/cli/create_db.py @@ -1,10 +1,6 @@ import argparse from murfey.util.db import clear, setup -from murfey.util.processing_db import ( - clear as processing_db_clear, - setup as processing_db_setup, -) def run(): @@ -18,24 +14,12 @@ def run(): action="store_false", help="Do not clear current database tables before creating specified tables", ) - parser.add_argument( - "--include-processing", - dest="processing", - default=True, - action="store_true", - help="Include processing results tables (MotionCorr, CTF, etc)", - ) args = parser.parse_args() from murfey.server.murfey_db import url - if args.clear and args.processing: - processing_db_clear(url()) - elif args.clear: + if args.clear: clear(url()) - if args.processing: - processing_db_setup(url()) - else: - setup(url()) + setup(url()) From 7ea39a7595d0035d580857326a72289c0684b3f1 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 26 Feb 2026 16:11:54 +0000 Subject: [PATCH 40/40] Remove processing db --- src/murfey/util/processing_db.py | 272 ------------------------------- tests/util/test_processing_db.py | 66 -------- 2 files changed, 338 deletions(-) delete mode 100644 src/murfey/util/processing_db.py delete mode 100644 tests/util/test_processing_db.py diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py deleted file mode 100644 index 24c2d33b8..000000000 --- a/src/murfey/util/processing_db.py +++ /dev/null @@ -1,272 +0,0 @@ -import datetime -from typing import List, Optional - -import sqlalchemy -from sqlmodel import Enum, Field, Relationship, create_engine - -from murfey.util import db - - -class MotionCorrection(db.SQLModel, table=True): # type: ignore - motionCorrectionId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="datacollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") - imageNumber: Optional[int] = None - firstFrame: Optional[int] = None - lastFrame: Optional[int] = None - dosePerFrame: Optional[float] = None - doseWeight: Optional[float] = None - totalMotion: Optional[float] = None - averageMotionPerFrame: Optional[float] = None - driftPlotFullPath: Optional[str] = None - micrographFullPath: Optional[str] = None - micrographSnapshotFullPath: Optional[str] = None - patchesUsedX: Optional[int] = None - patchesUsedY: Optional[int] = None - fftFullPath: Optional[str] = None - fftCorrectedFullPath: Optional[str] = None - comments: Optional[str] = None - movieId: Optional[int] = Field(foreign_key="movie.murfey_id") - auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( - back_populates="motion_correction" - ) - data_collection: Optional["db.DataCollection"] = Relationship( - back_populates="motion_correction" - ) - movie: Optional["db.Movie"] = Relationship(back_populates="motion_correction") - ctf: List["CTF"] = Relationship(back_populates="motion_correction") - particle_picker: List["ParticlePicker"] = Relationship( - back_populates="motion_correction" - ) - relative_ice_thickness: List["RelativeIceThickness"] = Relationship( - back_populates="motion_correction" - ) - - -class CTF(db.SQLModel, table=True): # type: ignore - ctfId: int = Field(primary_key=True, unique=True) - motionCorrectionId: Optional[int] = Field( - foreign_key="motioncorrection.motionCorrectionId" - ) - autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") - boxSizeX: Optional[float] = None - boxSizeY: Optional[float] = None - minResolution: Optional[float] = None - maxResolution: Optional[float] = None - minDefocus: Optional[float] = None - maxDefocus: Optional[float] = None - defocusStepSize: Optional[float] = None - astigmatism: Optional[float] = None - astigmatismAngle: Optional[float] = None - estimatedResolution: Optional[float] = None - estimatedDefocus: Optional[float] = None - amplitudeContrast: Optional[float] = None - ccValue: Optional[float] = None - fftTheoreticalFullPath: Optional[str] = None - iceRingDensity: Optional[float] = None - comments: Optional[str] = None - auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( - back_populates="ctf" - ) - motion_correction: Optional["MotionCorrection"] = Relationship(back_populates="ctf") - - -class ParticlePicker(db.SQLModel, table=True): # type: ignore - particlePickerId: int = Field(primary_key=True, unique=True) - programId: Optional[int] = Field(foreign_key="autoprocprogram.id") - firstMotionCorrectionId: Optional[int] = Field( - foreign_key="motioncorrection.motionCorrectionId" - ) - particlePickingTemplate: Optional[str] = None - particleDiameter: Optional[float] = None - numberOfParticles: Optional[int] = None - summaryImageFullPath: Optional[str] = None - motion_correction: Optional["MotionCorrection"] = Relationship( - back_populates="particle_picker" - ) - auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( - back_populates="particle_picker" - ) - particle_classification_group: List["ParticleClassificationGroup"] = Relationship( - back_populates="particle_picker" - ) - - -class Tomogram(db.SQLModel, table=True): # type: ignore - tomogramId: int = Field(primary_key=True, unique=True) - dataCollectionId: Optional[int] = Field(foreign_key="datacollection.id") - autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") - volumeFile: Optional[str] = None - stackFile: Optional[str] = None - sizeX: Optional[int] = None - sizeY: Optional[int] = None - sizeZ: Optional[int] = None - pixelSpacing: Optional[float] = None - residualErrorMean: Optional[float] = None - residualErrorSD: Optional[float] = None - xAxisCorrection: Optional[float] = None - tiltAngleOffset: Optional[float] = None - zShift: Optional[float] = None - fileDirectory: Optional[str] = None - centralSliceImage: Optional[str] = None - tomogramMovie: Optional[str] = None - xyShiftPlot: Optional[str] = None - projXY: Optional[str] = None - projXZ: Optional[str] = None - recordTimeStamp: Optional[datetime.datetime] = None - globalAlignmentQuality: Optional[float] = None - gridSquareId: Optional[int] = Field(foreign_key="searchmap.id") - pixelLocationX: Optional[int] = None - pixelLocationY: Optional[int] = None - auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( - back_populates="tomogram" - ) - data_collection: Optional["db.DataCollection"] = Relationship( - back_populates="tomogram" - ) - search_map: Optional["db.SearchMap"] = Relationship(back_populates="tomogram") - processed_tomogram: List["ProcessedTomogram"] = Relationship( - back_populates="tomogram" - ) - tilt_image_alignment: List["TiltImageAlignment"] = Relationship( - back_populates="tomogram" - ) - - -class ProcessedTomogram(db.SQLModel, table=True): # type: ignore - processedTomogramId: int = Field(primary_key=True, unique=True) - tomogramId: int = Field(foreign_key="tomogram.tomogramId") - filePath: Optional[str] = None - processingType: Optional[str] = None - tomogram: Optional["Tomogram"] = Relationship(back_populates="processed_tomogram") - - -class RelativeIceThickness(db.SQLModel, table=True): # type: ignore - relativeIceThicknessId: int = Field(primary_key=True, unique=True) - motionCorrectionId: Optional[int] = Field( - foreign_key="motioncorrection.motionCorrectionId" - ) - autoProcProgramId: Optional[int] = Field(foreign_key="autoprocprogram.id") - minimum: Optional[float] = None - q1: Optional[float] = None - median: Optional[float] = None - q3: Optional[float] = None - maximum: Optional[float] = None - auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( - back_populates="relative_ice_thickness" - ) - motion_correction: Optional["MotionCorrection"] = Relationship( - back_populates="relative_ice_thickness" - ) - - -class TiltImageAlignment(db.SQLModel, table=True): # type: ignore - movieId: int = Field(foreign_key="movie.murfey_id", primary_key=True) - tomogramId: int = Field(foreign_key="tomogram.tomogramId", primary_key=True) - defocusU: Optional[float] = None - defocusV: Optional[float] = None - psdFile: Optional[str] = None - resolution: Optional[float] = None - fitQuality: Optional[float] = None - refinedMagnification: Optional[float] = None - refinedTiltAngle: Optional[float] = None - refinedTiltAxis: Optional[float] = None - residualError: Optional[float] = None - movie: Optional["db.Movie"] = Relationship(back_populates="tilt_image_alignment") - tomogram: Optional["Tomogram"] = Relationship(back_populates="tilt_image_alignment") - - -class ParticleClassificationGroup(db.SQLModel, table=True): # type: ignore - particleClassificationGroupId: int = Field(primary_key=True, unique=True) - particlePickerId: Optional[int] = Field( - foreign_key="particlepicker.particlePickerId" - ) - programId: Optional[int] = Field(foreign_key="autoprocprogram.id") - type: Optional[str] = Enum("2D", "3D") - batchNumber: Optional[int] = None - numberOfParticlesPerBatch: Optional[int] = None - numberOfClassesPerBatch: Optional[int] = None - symmetry: Optional[str] = None - binnedPixelSize: Optional[float] = None - particle_picker: Optional["ParticlePicker"] = Relationship( - back_populates="particle_classification_group" - ) - auto_proc_program: Optional["db.AutoProcProgram"] = Relationship( - back_populates="particle_classification_group" - ) - particle_classification: List["ParticleClassification"] = Relationship( - back_populates="particle_classification_group" - ) - - -class ParticleClassification(db.SQLModel, table=True): # type: ignore - particleClassificationId: int = Field(primary_key=True, unique=True) - classNumber: Optional[int] = None - classImageFullPath: Optional[str] = None - particlesPerClass: Optional[int] = None - rotationAccuracy: Optional[float] = None - translationAccuracy: Optional[float] = None - estimatedResolution: Optional[float] = None - overallFourierCompleteness: Optional[float] = None - particleClassificationGroupId: Optional[int] = Field( - foreign_key="particleclassificationgroup.particleClassificationGroupId" - ) - classDistribution: Optional[float] = None - selected: Optional[int] = None - bFactorFitIntercept: Optional[float] = None - bFactorFitLinear: Optional[float] = None - bFactorFitQuadratic: Optional[float] = None - angularEfficiency: Optional[float] = None - suggestedTilt: Optional[float] = None - cryoem_initial_model: List["CryoemInitialModel"] = Relationship( - back_populates="particle_classification" - ) - particle_classification_group: Optional["ParticleClassificationGroup"] = ( - Relationship(back_populates="particle_classification") - ) - bfactor_fit: List["BFactorFit"] = Relationship( - back_populates="particle_classification" - ) - - -class BFactorFit(db.SQLModel, table=True): # type: ignore - bFactorFitId: int = Field(primary_key=True, unique=True) - particleClassificationId: int = Field( - foreign_key="particleclassification.particleClassificationId" - ) - resolution: Optional[float] = None - numberOfParticles: Optional[int] = None - particleBatchSize: Optional[int] = None - particle_classification: Optional["ParticleClassification"] = Relationship( - back_populates="bfactor_fit" - ) - - -class CryoemInitialModel(db.SQLModel, table=True): # type: ignore - cryoemInitialModelId: int = Field(primary_key=True, unique=True) - particleClassificationId: int = Field( - foreign_key="particleclassification.particleClassificationId" - ) - resolution: Optional[float] = None - numberOfParticles: Optional[int] = None - particle_classification: List["ParticleClassification"] = Relationship( - back_populates="cryoem_initial_model" - ) - - -""" -FUNCTIONS -""" - - -def setup(url: str): - engine = create_engine(url) - db.SQLModel.metadata.create_all(engine) - - -def clear(url: str): - engine = create_engine(url) - metadata = sqlalchemy.MetaData() - metadata.create_all(engine) - metadata.reflect(engine) - metadata.drop_all(engine) diff --git a/tests/util/test_processing_db.py b/tests/util/test_processing_db.py deleted file mode 100644 index 9dd2ef9fe..000000000 --- a/tests/util/test_processing_db.py +++ /dev/null @@ -1,66 +0,0 @@ -import os - -import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlmodel import Session, select - -from murfey.util.processing_db import ( - CTF, - BFactorFit, - CryoemInitialModel, - MotionCorrection, - ParticleClassification, - ParticleClassificationGroup, - ParticlePicker, - ProcessedTomogram, - RelativeIceThickness, - TiltImageAlignment, - Tomogram, -) - - -@pytest.fixture(scope="session") -def murfey_db_url(): - try: - return ( - f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}" - f"@{os.environ['POSTGRES_HOST']}:{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_DB']}" - ) - # Skip Murfey database-related tests if the environment for it hasn't been set up - except KeyError: - # If this fails in the GitHub test environment, raise it as a genuine error - if os.getenv("GITHUB_ACTIONS") == "true": - raise KeyError - pytest.skip("Murfey PostgreSQL database has not been set up; skipping test") - return "" - - -def test_processing_tables_exist(murfey_db_url): - from sqlmodel import SQLModel - - engine = create_engine(murfey_db_url) - SQLModel.metadata.create_all(engine) - connection = engine.connect() - - with sessionmaker( - bind=connection, expire_on_commit=False, class_=Session - )() as murfey_db_session: - assert murfey_db_session.exec(select(MotionCorrection)).all() == [] - assert murfey_db_session.exec(select(CTF)).all() == [] - assert murfey_db_session.exec(select(ParticlePicker)).all() == [] - assert murfey_db_session.exec(select(Tomogram)).all() == [] - assert murfey_db_session.exec(select(ProcessedTomogram)).all() == [] - assert murfey_db_session.exec(select(RelativeIceThickness)).all() == [] - assert murfey_db_session.exec(select(TiltImageAlignment)).all() == [] - assert murfey_db_session.exec(select(ParticleClassificationGroup)).all() == [] - assert murfey_db_session.exec(select(ParticleClassification)).all() == [] - assert murfey_db_session.exec(select(BFactorFit)).all() == [] - assert murfey_db_session.exec(select(CryoemInitialModel)).all() == [] - - from murfey.util.db import ClientEnvironment, DataCollectionGroup - - assert murfey_db_session.exec(select(ClientEnvironment)).all() == [] - assert murfey_db_session.exec(select(DataCollectionGroup)).all() == [] - - connection.close()