from great_tables import GT, md, html, style, loc, px
import polars as pl
import mdpd
from zfish.plot.plot_commons import cmaps
MONOFONT = "source code pro"
TABLEFONT = None
def gt_render_pdf(
df,
header="Base Dimensions",
hide=None,
font=MONOFONT,
weith="normal",
size=px(14),
columns=(),
):
gt_df = (
GT(df)
.tab_style(
style=style.text(
color=cmaps.spatial_element_dark[3],
font=MONOFONT,
weight="normal",
size=size,
),
locations=loc.body(columns=list(columns)),
)
.tab_header(
title=md(header),
)
)
if hide is not None:
return gt_df.cols_hide(hide)
return gt_df
def gt_render(
df,
header="Base Dimensions",
hide=None,
font=MONOFONT,
weith="normal",
size=px(14),
columns=(),
v_padding=0.8,
h_padding=0.8,
):
gt_df = (
GT(df)
.tab_style(
style=style.text(
color=cmaps.spatial_element_dark[3],
font=MONOFONT,
weight="normal",
size=size,
),
locations=loc.body(columns=list(columns)),
)
.tab_header(
title=md(header),
)
.opt_vertical_padding(scale=v_padding)
.opt_horizontal_padding(scale=h_padding)
)
if hide is not None:
return gt_df.cols_hide(hide)
return gt_df
def gt_render_from_markdown(
df,
header="Base Dimensions",
hide=None,
size=px(14),
columns=(),
v_padding=0.8,
h_padding=0.8,
):
gt_df = (
GT(df)
.tab_header(
title=md(header),
)
.opt_vertical_padding(scale=v_padding)
.opt_horizontal_padding(scale=h_padding)
)
if hide is not None:
return gt_df.cols_hide(hide)
return gt_df.fmt_markdown(list(columns))In [2]:
In [3]:
import polars as pl
import mdpd
df_data_structures = pl.DataFrame(
mdpd.from_md("""
| Data Structure | Description |
|------------------------|-----------------------------------------------------------|
| **Raster Data** | |
| Image | Fluorescence intensity measurements sampled on a regular grid. |
| Binary Mask | Partition of an image into foreground and background, representing a single object. |
| Label Image | Partitioning of an image into *n* objects with integer IDs (labels). `0` for background. Represents a collection of objects. |
| Distance Transform | Image with pixel values storing the distance to a binary mask. |
| Multiscale Pyramid | Collection of images sampled at different scales (pyramid schedule) with shared coverage. |
| **Vector Data** | |
| Point / Vertex | Position in space, potentially associated with an object or features: `(z, y, x)`. |
| Vector | *Difference* between two points, or direction and magnitude: `(dz, dy, dx)`. |
| Line Segment / Edge | 1D path between two points (also 1-simplex): `(z, y, x), (z, y, x)` *or* `(z, y, x), (dz, dy, dx)`. |
| Face | Simple 2D surface, *often* a triangle. Defined by points and their connectivity. |
| Mesh | Representation of objects or surfaces using points, edges, and faces placed in physical space. |
| Graph (Spatial) | Mesh with only points and edges to represent objects in space and their relationships. |
| Neighborhood | Spatial graph representing object neighborhoods, for example using sparse arrays. |
| **Non-Spatial**\* | |
| Feature Table | Quantitative or categorical features (columns) associated with a set of elements (rows). |
| Graph | Nodes and edges representing entities and relationships between them. Many possible representations, *usually* sparse. |
| Hierarchy Table | Graph encoding relationships (rows) between pairs of objects (columns; 2 IDs). Edge list representation of a graph. |
""")
)
def gt_bold_column_labels(df):
return (
GT(df_data_structures)
.fmt_markdown()
.cols_label(
**{col: md(f"**{col}**") for col in df.columns},
)
)In [4]:
gt_bold_column_labels(df_data_structures).tab_source_note(
source_note=md("*Since all our measurements are derived from pixels they can *always* be associated with spatial objects. But we often drop the spatial data components, like when doing regression modelling with extracted features, and think of them as non-spatial.")
)| Data Structure | Description |
|---|---|
| Raster Data | |
| Image | Fluorescence intensity measurements sampled on a regular grid. |
| Binary Mask | Partition of an image into foreground and background, representing a single object. |
| Label Image | Partitioning of an image into n objects with integer IDs (labels). 0 for background. Represents a collection of objects. |
| Distance Transform | Image with pixel values storing the distance to a binary mask. |
| Multiscale Pyramid | Collection of images sampled at different scales (pyramid schedule) with shared coverage. |
| Vector Data | |
| Point / Vertex | Position in space, potentially associated with an object or features: (z, y, x). |
| Vector | Difference between two points, or direction and magnitude: (dz, dy, dx). |
| Line Segment / Edge | 1D path between two points (also 1-simplex): (z, y, x), (z, y, x) or (z, y, x), (dz, dy, dx). |
| Face | Simple 2D surface, often a triangle. Defined by points and their connectivity. |
| Mesh | Representation of objects or surfaces using points, edges, and faces placed in physical space. |
| Graph (Spatial) | Mesh with only points and edges to represent objects in space and their relationships. |
| Neighborhood | Spatial graph representing object neighborhoods, for example using sparse arrays. |
| Non-Spatial* | |
| Feature Table | Quantitative or categorical features (columns) associated with a set of elements (rows). |
| Graph | Nodes and edges representing entities and relationships between them. Many possible representations, usually sparse. |
| Hierarchy Table | Graph encoding relationships (rows) between pairs of objects (columns; 2 IDs). Edge list representation of a graph. |
| *Since all our measurements are derived from pixels they can always be associated with spatial objects. But we often drop the spatial data components, like when doing regression modelling with extracted features, and think of them as non-spatial. | |
In [5]:
df_dims_base = pl.DataFrame(
mdpd.from_md("""
| Type | Dimension | Coordinate | Coordinate Description |
| :---|:--|:-------------|:------------|
| nominal | `roi` | `['site0', 'site1', ..., 'site211']` | Named, bounded regions of space. |
| temporal | `t` | `[0.0, 15.0, 30.0, ..., 1000.0] s` | Acquisition timestamps. |
| spatial | `z` | `[0.0, 271.0] μm` | Bounded continuous z-coordinate. |
| spatial | `y` | `[0.0, 650.0] μm` | Bounded continuous y-coordinate. |
| spatial | `x` | `[0.0, 650.0] μm` | Bounded continuous x-coordinate. |
| ordinal | `m` | `[0, 1, 2, 3, 4, 5]` | Sampling scales of the pyramid grids. |
| nominal | `c` | `['DAPI.0', 'PCNA.0', ..., 'Nanog.3']` | Molecular targets with acquisition cycle. |
| ratio | `val` | `[0.0, 4.2, 5.4, ...] AU` | Intensity histogram for a channel. |
| nominal | `o` | `['embryo', 'cell', ..., 'nuc']` | Types of segmented objects. |
| nominal | `lbl`* | `[1, 2, ... n]` | Instance labels of an object type. |
""")
)In [6]:
tbl = (
gt_render_from_markdown(df_dims_base, columns=("Dimension", "Coordinate"))
.cols_move_to_start("Dimension")
.cols_label({"Dimension": "Dim"})
)
tbl| Base Dimensions | |||
|---|---|---|---|
| Dim | Type | Coordinate | Coordinate Description |
roi |
nominal | ['site0', 'site1', ..., 'site211'] |
Named, bounded regions of space. |
t |
temporal | [0.0, 15.0, 30.0, ..., 1000.0] s |
Acquisition timestamps. |
z |
spatial | [0.0, 271.0] μm |
Bounded continuous z-coordinate. |
y |
spatial | [0.0, 650.0] μm |
Bounded continuous y-coordinate. |
x |
spatial | [0.0, 650.0] μm |
Bounded continuous x-coordinate. |
m |
ordinal | [0, 1, 2, 3, 4, 5] |
Sampling scales of the pyramid grids. |
c |
nominal | ['DAPI.0', 'PCNA.0', ..., 'Nanog.3'] |
Molecular targets with acquisition cycle. |
val |
ratio | [0.0, 4.2, 5.4, ...] AU |
Intensity histogram for a channel. |
o |
nominal | ['embryo', 'cell', ..., 'nuc'] |
Types of segmented objects. |
lbl* |
nominal | [1, 2, ... n] |
Instance labels of an object type. |
In [7]:
df_dims_compound = pl.DataFrame(
mdpd.from_md("""
| Type | Dimension | Components | Description |
|:------------|:---------------|:-------------------|:----------------------------------------------------|
| categorical | `image` | `roi`, `m`, `c` | Single-channel, single-scale image. |
| categorical | `label_image` | `roi`, `m`, `o` | Single-scale label image. |
| categorical | `label_object` | `roi`, `o`, `lbl` | Segmented object (ID is valid across `m`). |
""")
)In [8]:
gt_render_from_markdown(
df_dims_compound.drop('Type'),
# hide="Type",
header="Compound Dimensions",
columns=("Dimension", "Components"),
h_padding=1.8,
).cols_move_to_start("Dimension")| Compound Dimensions | ||
|---|---|---|
| Dimension | Components | Description |
image |
roi, m, c |
Single-channel, single-scale image. |
label_image |
roi, m, o |
Single-scale label image. |
label_object |
roi, o, lbl |
Segmented object (ID is valid across `m`). |
In [9]:
df_dims_slice = pl.DataFrame(
mdpd.from_md("""
| Dimension | Components | Description |
|---------------------|--------------------------------------------|--------------------------------------------------|
| `mc_image` | `roi`, `m`, `c:n` | Multi-channel, single-scale image. |
| `image_pyramid` | `roi`, `m:n`, `c` | Single-channel, multi-scale image. |
| `mc_image_pyramid` | `roi`, `m:n`, `c:n` | Multi-channel, multi-scale image. |
| `sub_label_image` | `roi`, `m`, `o`, `lbl:n` | Label image formed from a subset of labels. |
| `voxel_region` | `roi`, `m`, `iz`, `iy`, `ix` | Voxel without associated data. |
| `mc_mo_voxel` | `roi`, `m`, `iz`, `iy`, `ix`, `c:n`, `o:n` | Voxel with channel data and object type data. |
| `roi` | `roi`, `m:n`, `c:n`, `o:n` | Region of interest images and label images. |
| `dataset` | `roi:n`, `m:n`, `c:n` `o:n` | Dataset containing a collection of ROIs. |
""")
)
df_dims_slice
shape: (8, 3)
| Dimension | Components | Description |
|---|---|---|
| str | str | str |
| "`mc_image`" | "`roi`, `m`, `c:n`" | "Multi-channel, single-scale im… |
| "`image_pyramid`" | "`roi`, `m:n`, `c`" | "Single-channel, multi-scale im… |
| "`mc_image_pyramid`" | "`roi`, `m:n`, `c:n`" | "Multi-channel, multi-scale ima… |
| "`sub_label_image`" | "`roi`, `m`, `o`, `lbl:n`" | "Label image formed from a subs… |
| "`voxel_region`" | "`roi`, `m`, `iz`, `iy`, `ix`" | "Voxel without associated data." |
| "`mc_mo_voxel`" | "`roi`, `m`, `iz`, `iy`, `ix`, … | "Voxel with channel data and ob… |
| "`roi`" | "`roi`, `m:n`, `c:n`, `o:n`" | "Region of interest images and … |
| "`dataset`" | "`roi:n`, `m:n`, `c:n` `o:n`" | "Dataset containing a collectio… |
In [10]:
gt_render_from_markdown(
df_dims_slice,
header="Slice Dimensions",
columns=["Dimension", "Components"],
h_padding=0.8,
).cols_move_to_start("Dimension")| Slice Dimensions | ||
|---|---|---|
| Dimension | Components | Description |
mc_image |
roi, m, c:n |
Multi-channel, single-scale image. |
image_pyramid |
roi, m:n, c |
Single-channel, multi-scale image. |
mc_image_pyramid |
roi, m:n, c:n |
Multi-channel, multi-scale image. |
sub_label_image |
roi, m, o, lbl:n |
Label image formed from a subset of labels. |
voxel_region |
roi, m, iz, iy, ix |
Voxel without associated data. |
mc_mo_voxel |
roi, m, iz, iy, ix, c:n, o:n |
Voxel with channel data and object type data. |
roi |
roi, m:n, c:n, o:n |
Region of interest images and label images. |
dataset |
roi:n, m:n, c:n o:n |
Dataset containing a collection of ROIs. |
In [11]:
df_features_overview = pl.DataFrame(
mdpd.from_md("""
| Feature Type | Object ID | Resources | Features | Examples |
|--------------------|-------------------------------|--------------------------------------------|------------------|--------------------------------------------------|
| Label | `roi`, `o`, `lbl` | `m` | `f:label` | Volume, Roundness, Principal Axes, Centroid Position |
| Intensity | `roi`, `o`, `lbl` | `m`, `c` | `f:intensity` | DAPI Abundance*, Concentration**, Variance |
| Colocalization | `roi`, `o`, `lbl` | `m`, `c.0`, `c.1` | `f:coloc` | DAPI Correlation |
| Distance | `roi`, `o`, `lbl` | `m`, `o.to`, `[lbl.to]`, `dist_transf` | `f:distance` | Centroid Distance To Embryo Border |
| Hierarchy (1:1) | `roi`, `o`, `lbl` | `o.from`, `f:in` | `f:in` | Nuclear DAPI Abundance in Cell |
| Hierarchy (m:1) | `roi`, `o`, `lbl` | `o.from`, `agg_func`, `[f:in]` | `Count`, `f:in` | Mean Nuclear Volume in Embryo |
| Neighborhood | `roi`, `o`, `lbl` | `nhd_query`, `agg_func`, `[f:in]` | `Count`, `f:in` | Cell Count in Radius 30 μm Neighborhood |
| Embedding | `roi`, `o`, `lbl` | `model`, `f:in` | `f:out` | UMAP Embedding, Cell Cycle Phase, Cell Type |
""")
)
df_features_overview#.pipe(GT).fmt_markdown().tab_source_note(source_note="*Sum / **Mean intensity")
shape: (8, 5)
| Feature Type | Object ID | Resources | Features | Examples |
|---|---|---|---|---|
| str | str | str | str | str |
| "Label" | "`roi`, `o`, `lbl`" | "`m`" | "`f:label`" | "Volume, Roundness, Principal A… |
| "Intensity" | "`roi`, `o`, `lbl`" | "`m`, `c`" | "`f:intensity`" | "DAPI Abundance*, Concentration… |
| "Colocalization" | "`roi`, `o`, `lbl`" | "`m`, `c.0`, `c.1`" | "`f:coloc`" | "DAPI Correlation" |
| "Distance" | "`roi`, `o`, `lbl`" | "`m`, `o.to`, `[lbl.to]`, `dist… | "`f:distance`" | "Centroid Distance To Embryo Bo… |
| "Hierarchy (1:1)" | "`roi`, `o`, `lbl`" | "`o.from`, `f:in`" | "`f:in`" | "Nuclear DAPI Abundance in Cell" |
| "Hierarchy (m:1)" | "`roi`, `o`, `lbl`" | "`o.from`, `agg_func`, `[f:in]`" | "`Count`, `f:in`" | "Mean Nuclear Volume in Embryo" |
| "Neighborhood" | "`roi`, `o`, `lbl`" | "`nhd_query`, `agg_func`, `[f:i… | "`Count`, `f:in`" | "Cell Count in Radius 30 μm Nei… |
| "Embedding" | "`roi`, `o`, `lbl`" | "`model`, `f:in`" | "`f:out`" | "UMAP Embedding, Cell Cycle Pha… |
In [12]:
gt_render_from_markdown(
df_features_overview,
header="Feature Tables",
columns=("Object ID", "Resources", "Features"),
# h_padding=0.8,
).tab_source_note(source_note="*Sum / **Mean intensity")| Feature Tables | ||||
|---|---|---|---|---|
| Feature Type | Object ID | Resources | Features | Examples |
| Label | roi, o, lbl |
m |
f:label |
Volume, Roundness, Principal Axes, Centroid Position |
| Intensity | roi, o, lbl |
m, c |
f:intensity |
DAPI Abundance*, Concentration**, Variance |
| Colocalization | roi, o, lbl |
m, c.0, c.1 |
f:coloc |
DAPI Correlation |
| Distance | roi, o, lbl |
m, o.to, [lbl.to], dist_transf |
f:distance |
Centroid Distance To Embryo Border |
| Hierarchy (1:1) | roi, o, lbl |
o.from, f:in |
f:in |
Nuclear DAPI Abundance in Cell |
| Hierarchy (m:1) | roi, o, lbl |
o.from, agg_func, [f:in] |
Count, f:in |
Mean Nuclear Volume in Embryo |
| Neighborhood | roi, o, lbl |
nhd_query, agg_func, [f:in] |
Count, f:in |
Cell Count in Radius 30 μm Neighborhood |
| Embedding | roi, o, lbl |
model, f:in |
f:out |
UMAP Embedding, Cell Cycle Phase, Cell Type |
| *Sum / **Mean intensity | ||||
In [13]:
FEATURE_SETS = {
"ccp": ("CCP",),
"ccp-feats": (
"PCNA",
"DAPI",
"n_Volume",
"n.cy_VolumeRatio",
"n_Roundness",
"n_Flatness",
),
"zga": ("Pou5", "Nanog", "H3K27Ac", "H2B"),
"s5p": ("Pol2.S5P",),
}
(
GT(
pl.DataFrame(
[("", ["Pol2.S2P"])]
+ list(FEATURE_SETS.items())
+ [
("ccp-spline", ["bs(CCP, 3, fit_intercept=False)"]),
("ccp-asympt", ["CCP(a, b, c)"]),
("ccp-power", ["CCP(a, b)"]),
],
orient="row",
schema=["Component", "Features"],
)
.with_columns(
pl.Series(
"Description",
[
"Pol-II-S2P (elongating)",
"Interphase CCP",
"Cell cycle phase features*",
"ZGA factors",
"Pol-II-S5P (initiating)",
"B-spline encoded interphase CCP (3 knots w/o intercept)",
"Asymptotic regression on CCP",
"Power curve regression on CCP",
],
),
)
.with_columns(pl.col("Features").list.join(", "))
.with_columns(
pl.Series(
"category",
[
"regression target",
"linear predictors",
"linear predictors",
"linear predictors",
"linear predictors",
"non-linear predictors",
"non-linear predictors",
"non-linear predictors",
],
)
)
)
.tab_source_note(
source_note="*a subset of the features that went into the CCP model."
)
.tab_stub(rowname_col="Component", groupname_col="category")
.tab_stubhead("Component")
.tab_style(
style=style.text(
color=cmaps.spatial_element_dark[3],
font=MONOFONT,
weight="normal",
),
locations=loc.body(columns=["Component"]),
)
.tab_options(column_labels_font_weight="bold")
# .opt_stylize(style=2, color='blue')
)| Component | Features | Description |
|---|---|---|
| regression target | ||
| Pol2.S2P | Pol-II-S2P (elongating) | |
| linear predictors | ||
| ccp | CCP | Interphase CCP |
| ccp-feats | PCNA, DAPI, n_Volume, n.cy_VolumeRatio, n_Roundness, n_Flatness | Cell cycle phase features* |
| zga | Pou5, Nanog, H3K27Ac, H2B | ZGA factors |
| s5p | Pol2.S5P | Pol-II-S5P (initiating) |
| non-linear predictors | ||
| ccp-spline | bs(CCP, 3, fit_intercept=False) | B-spline encoded interphase CCP (3 knots w/o intercept) |
| ccp-asympt | CCP(a, b, c) | Asymptotic regression on CCP |
| ccp-power | CCP(a, b) | Power curve regression on CCP |
| *a subset of the features that went into the CCP model. | ||
In [14]:
import polars as pl
from great_tables import GT
df_ccp_components = pl.DataFrame(
{
"Component": [
"`PolarsSelector`",
"`StratifiedTransformer`",
"`StratifiedEmbedderAligned`",
"`CcpTransformer`",
],
"Description": [
"Selects columns from a Polars DataFrame using a column selector.",
"Meta-transformer that maps a transformer across subsets of the data by fitting one for each subset.",
"Meta-transformer that maps an embedding (e.g., UMAP) on subsets of data and aligns the results using a rigid transformation.",
"Cell cycle phase transformer that takes a pre-embedding as input, fits a principal circle, and returns `CCP`, `NormalizedCCP`, and `DistanceToCircleCCP`."
],
}
)
gt_render_from_markdown(
df_ccp_components,
header="Scikit-learn Compatible CCP Components",
columns=("Component", "Description"),
h_padding=0.8,
)| Scikit-learn Compatible CCP Components | |
|---|---|
| Component | Description |
PolarsSelector |
Selects columns from a Polars DataFrame using a column selector. |
StratifiedTransformer |
Meta-transformer that maps a transformer across subsets of the data by fitting one for each subset. |
StratifiedEmbedderAligned |
Meta-transformer that maps an embedding (e.g., UMAP) on subsets of data and aligns the results using a rigid transformation. |
CcpTransformer |
Cell cycle phase transformer that takes a pre-embedding as input, fits a principal circle, and returns CCP, NormalizedCCP, and DistanceToCircleCCP. |
In [22]:
df_ab_stain_conditions
shape: (12, 9)
| acquisition | wavelength | target_primary | host_primary | dilution_primary | secondary_is_fab | name_secondary | dilution_secondary | stain_type |
|---|---|---|---|---|---|---|---|---|
| i64 | i64 | str | str | str | bool | str | str | str |
| 0 | 488 | "Pol-II-S2P" | "Rat" | "1:250" | false | "d@rt 488" | "1:200" | "Antibody" |
| 0 | 568 | "FLAG" | "Mouse" | "1:500" | false | "d@ms 568" | "1:200" | "Antibody" |
| 0 | 647 | "PCNA" | "Rabbit" | "1:250" | false | "d@rb 647" | "1:200" | "Antibody" |
| 1 | 488 | "H3K27Ac" | "Rabbit" | "1:500" | true | "Zenon Rb 488" | "1:100" | "Antibody" |
| 1 | 568 | "bCatenin" | "Mouse" | "1:250" | false | "d@ms 568" | "1:400" | "Antibody" |
| … | … | … | … | … | … | … | … | … |
| 2 | 568 | "ALYREF" | "Mouse" | "1:250" | true | "Zenon Ms 555" | "1:100" | "Antibody" |
| 2 | 647 | "Pol-II-S5P" | "Rat" | "1:250" | false | "d@rt 647" | "1:200" | "Antibody" |
| 3 | 488 | "Nanog" | "Rabbit" | "1:280" | true | "Zenon Rb 488" | "1:28" | "Antibody" |
| 3 | 568 | "YAP" | "Mouse" | "1:50" | false | "d@ms 568" | "1:400" | "Antibody" |
| 3 | 647 | "XRN2" | "Rabbit" | "1:100" | true | "Zenon Rb 647" | "1:20" | "Antibody" |
In [1]:
df_ab_stain_conditions = pl.read_csv("../Data/AB_stainings.csv")
df_ab_stain_conditions.select(
pl.col("target_primary").alias("primary"),
pl.col("host_primary").alias("host"),
pl.col("dilution_primary").alias("dilution"),
pl.col('name_secondary').alias('secondary'),
pl.col('dilution_secondary').alias('secondary dilution'),
pl.when(pl.col('secondary_is_fab')).then(pl.lit('IF direct')).otherwise(pl.lit('IF indirect')).alias('stain type'),
).pipe(GT)| primary | host | dilution | stain type | secondary name | secondary dilution |
|---|---|---|---|---|---|
| Pol-II-S2P | Rat | 1:250 | IF indirect | d@rt 488 | 1:200 |
| FLAG | Mouse | 1:500 | IF indirect | d@ms 568 | 1:200 |
| PCNA | Rabbit | 1:250 | IF indirect | d@rb 647 | 1:200 |
| H3K27Ac | Rabbit | 1:500 | IF direct | Zenon Rb 488 | 1:100 |
| bCatenin | Mouse | 1:250 | IF indirect | d@ms 568 | 1:400 |
| pH3 | Rabbit | 1:150 | IF direct | Zenon Rb 647 | 1:150 |
| H2B | Rabbit | 1:100 | IF direct | Zenon Rb 488 | 1:250 |
| ALYREF | Mouse | 1:250 | IF direct | Zenon Ms 555 | 1:100 |
| Pol-II-S5P | Rat | 1:250 | IF indirect | d@rt 647 | 1:200 |
| Nanog | Rabbit | 1:280 | IF direct | Zenon Rb 488 | 1:28 |
| YAP | Mouse | 1:50 | IF indirect | d@ms 568 | 1:400 |
| XRN2 | Rabbit | 1:100 | IF direct | Zenon Rb 647 | 1:20 |