Copying MuData objects

[gtca]

Hi! I am wondering if there is a way to copy MuData objects (i.e. MuData_copy = MuData.copy()) without getting TypeError: cannot unpack non-iterable NoneType object

Originally posted by @SamuelAMiller1 in #24 (comment)

[gtca]

The minimal example does work:

import numpy as np
import muon as mu

x = mu.AnnData(X=np.random.normal(size=1000).reshape(-1, 100))
y = mu.AnnData(X=np.random.normal(size=2000).reshape(-1, 100))

md = mu.MuData({"x": x, "y": y})
md_copy = md.copy()

Is there a piece of code that could help to reproduce the TypeError, @SamuelAMiller1?

[SamuelAMiller1]

Sure, this is is the simplest example I can produce that generates the error.

import scanpy as sc
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib as mpl
import muon as mu
from muon import MuData

sc.logging.print_versions()
sc.set_figure_params(frameon=False, figsize=(4, 4))

!mkdir -p data
!wget http://cf.10xgenomics.com/samples/cell-exp/3.1.0/5k_pbmc_protein_v3_nextgem/5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5 -O data/5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5

datafile = "data/5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5"

pbmc = sc.read_10x_h5(datafile, gex_only=False)
pbmc.var_names_make_unique()
pbmc.layers["counts"] = pbmc.X.copy()
sc.pp.filter_genes(pbmc, min_counts=1)
pbmc

protein = pbmc[:, pbmc.var["feature_types"] == "Antibody Capture"].copy()
rna = pbmc[:, pbmc.var["feature_types"] == "Gene Expression"].copy()

mdata = MuData({"rna": rna, "prot": protein})

mdata_copy = mdata.copy()

This is the complete error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-18-052cabbf3e25> in <module>
----> 1 mdata_copy = mdata.copy()

/srv/conda/envs/saturn/lib/python3.8/site-packages/muon/_core/mudata.py in copy(self, filename)
    293                 self.obsm.copy(),
    294                 self.varm.copy(),
--> 295                 deepcopy(self.obsp),
    296                 deepcopy(self.varp),
    297                 self.obsmap.copy(),

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--> 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/srv/conda/envs/saturn/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--> 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--> 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/srv/conda/envs/saturn/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--> 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--> 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/srv/conda/envs/saturn/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--> 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--> 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/srv/conda/envs/saturn/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--> 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--> 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/srv/conda/envs/saturn/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--> 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--> 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/srv/conda/envs/saturn/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--> 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--> 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/srv/conda/envs/saturn/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--> 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    170                     y = x
    171                 else:
--> 172                     y = _reconstruct(x, memo, *rv)
    173 
    174     # If is its own copy, don't memoize.

/srv/conda/envs/saturn/lib/python3.8/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    268     if state is not None:
    269         if deep:
--> 270             state = deepcopy(state, memo)
    271         if hasattr(y, '__setstate__'):
    272             y.__setstate__(state)

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--> 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/srv/conda/envs/saturn/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--> 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    144     copier = _deepcopy_dispatch.get(cls)
    145     if copier is not None:
--> 146         y = copier(x, memo)
    147     else:
    148         if issubclass(cls, type):

/srv/conda/envs/saturn/lib/python3.8/copy.py in _deepcopy_dict(x, memo, deepcopy)
    228     memo[id(x)] = y
    229     for key, value in x.items():
--> 230         y[deepcopy(key, memo)] = deepcopy(value, memo)
    231     return y
    232 d[dict] = _deepcopy_dict

/srv/conda/envs/saturn/lib/python3.8/copy.py in deepcopy(x, memo, _nil)
    151             copier = getattr(x, "__deepcopy__", None)
    152             if copier is not None:
--> 153                 y = copier(memo)
    154             else:
    155                 reductor = dispatch_table.get(cls)

/srv/conda/envs/saturn/lib/python3.8/site-packages/anndata/_core/views.py in __deepcopy__(self, memo)
     55     # TODO: This makes `deepcopy(obj)` return `obj._view_args.parent._adata_ref`, fix it
     56     def __deepcopy__(self, memo):
---> 57         parent, attrname, keys = self._view_args
     58         return deepcopy(getattr(parent._adata_ref, attrname))
     59 

TypeError: cannot unpack non-iterable NoneType object

[ilia-kats]

Looks like a duplicate of #30. Can you try with master?

[SamuelAMiller1]

Same error with master.

[gtca]

I could not reproduce this error with the master, it seems to work fine.
I tested it with Python 3.8 in a few environments. Here is an example of what sc.logging.print_header() outputs in one of them:

scanpy==1.7.1 anndata==0.7.6 umap==0.4.6 numpy==1.21.0 scipy==1.7.0 pandas==1.2.3 
scikit-learn==0.23.1 statsmodels==0.11.1 python-igraph==0.8.2 louvain==0.7.0 leidenalg==0.8.2

By the way, muon comes with readers so one can just do this to get a MuData object:

pbmc = mu.read_10x_h5(datafile)

pbmc
# MuData object with n_obs × n_vars = 5527 × 33570
#   2 modalities
#     rna:        5527 x 33538
#       var:      'gene_ids', 'feature_types', 'genome'
#     prot:       5527 x 32
#       var:      'gene_ids', 'feature_types', 'genome'

[SamuelAMiller1]

Running from Jupyter Notebook

Python 3.8.10 | packaged by conda-forge

Reinstalled those versions to match your sc.logging.print_header() output but still getting the error. Long shot, but are you using a different release version of python and may the issue be related?

[SamuelAMiller1]

Update: I do not get the error in a different environment, so there must be some conflict in my current environment. Thanks for testing.

[gtca]

I also made a Docker image to test it, and it seems to work for executing the lines above:

FROM python:3.9.6-buster

RUN apt-get update
RUN apt-get install -y make automake gcc g++ subversion python3-dev
RUN apt-get install -y gfortran musl libblas-dev liblapack-dev
RUN apt-get install -y git

RUN pip install git+https://github.com/pmbio/muon

CMD []