Skip to content

cds2.io

Pandas-backed data I/O.

I/O — convenience re-export of pandas read/write; native value is summarize/iter_csv.

read_csv(path, **kwargs)

Read a CSV file into a DataFrame.

Source code in src/cds2/io.py 
24
25
26
def read_csv(path: str, **kwargs: Any) -> pd.DataFrame:
    """Read a CSV file into a DataFrame."""
    return cast("pd.DataFrame", pd.read_csv(path, **kwargs))

write_csv(df, path, index=False, **kwargs)

Write a DataFrame to CSV.

Source code in src/cds2/io.py 
29
30
31
32
def write_csv(df: pd.DataFrame, path: str, index: bool = False, **kwargs: Any) -> str:
    """Write a DataFrame to CSV."""
    df.to_csv(path, index=index, **kwargs)
    return path

read_json(path, **kwargs)

Read records-oriented JSON into a DataFrame.

Source code in src/cds2/io.py 
35
36
37
def read_json(path: str, **kwargs: Any) -> pd.DataFrame:
    """Read records-oriented JSON into a DataFrame."""
    return cast("pd.DataFrame", pd.read_json(path, **kwargs))

write_json(df, path, orient='records', **kwargs)

Write a DataFrame to JSON.

Source code in src/cds2/io.py 
40
41
42
43
def write_json(df: pd.DataFrame, path: str, orient: str = "records", **kwargs: Any) -> str:
    """Write a DataFrame to JSON."""
    df.to_json(path, orient=cast(Any, orient), **kwargs)
    return path

read_excel(path, **kwargs)

Read the first sheet of an Excel workbook (requires openpyxl).

Source code in src/cds2/io.py 
46
47
48
49
50
51
52
53
def read_excel(path: str, **kwargs: Any) -> pd.DataFrame:
    """Read the first sheet of an Excel workbook (requires openpyxl)."""
    try:
        import openpyxl  # noqa: F401
    except ImportError as exc:
        msg = "Excel support requires the optional 'openpyxl' package"
        raise RuntimeError(msg) from exc
    return cast("pd.DataFrame", pd.read_excel(path, **kwargs))

write_excel(df, path, index=False, **kwargs)

Write a DataFrame to an Excel workbook (requires openpyxl).

Source code in src/cds2/io.py 
56
57
58
59
60
61
62
63
64
def write_excel(df: pd.DataFrame, path: str, index: bool = False, **kwargs: Any) -> str:
    """Write a DataFrame to an Excel workbook (requires openpyxl)."""
    try:
        import openpyxl  # noqa: F401
    except ImportError as exc:
        msg = "Excel support requires the optional 'openpyxl' package"
        raise RuntimeError(msg) from exc
    df.to_excel(path, index=index, **kwargs)
    return path

read_parquet(path, **kwargs)

Read a Parquet file (requires pyarrow or fastparquet).

Source code in src/cds2/io.py 
67
68
69
def read_parquet(path: str, **kwargs: Any) -> pd.DataFrame:
    """Read a Parquet file (requires pyarrow or fastparquet)."""
    return pd.read_parquet(path, **kwargs)

write_parquet(df, path, index=False, **kwargs)

Write a Parquet file (requires pyarrow or fastparquet).

Source code in src/cds2/io.py 
72
73
74
75
def write_parquet(df: pd.DataFrame, path: str, index: bool = False, **kwargs: Any) -> str:
    """Write a Parquet file (requires pyarrow or fastparquet)."""
    df.to_parquet(path, index=index, **kwargs)
    return path

summarize(df)

Column-level summary: dtypes, nulls, uniques and numeric moments.

Source code in src/cds2/io.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def summarize(df: pd.DataFrame) -> pd.DataFrame:
    """Column-level summary: dtypes, nulls, uniques and numeric moments."""
    non_null = df.notna().sum()
    nulls = len(df) - non_null
    unique_counts = df.nunique(dropna=True)
    numeric = df.select_dtypes(include=[np.number])
    moments = (
        numeric.agg(["mean", "std", "min", "max"])
        if len(numeric.columns)
        else pd.DataFrame(index=["mean", "std", "min", "max"])
    )
    rows: list[dict[str, Any]] = []
    for column in df.columns:
        mean_value: float | None = None
        std_value: float | None = None
        min_value: float | None = None
        max_value: float | None = None
        if column in moments.columns:
            mean_value, std_value, min_value, max_value = (float(v) for v in moments[column])
        rows.append(
            {
                "column": column,
                "dtype": str(df[column].dtype),
                "non_null": int(non_null[column]),
                "nulls": int(nulls[column]),
                "unique": int(unique_counts[column]),
                "mean": mean_value,
                "std": std_value,
                "min": min_value,
                "max": max_value,
            }
        )
    result: pd.DataFrame = pd.DataFrame(rows)
    return result

iter_csv(path, chunksize=100000, **kwargs)

Yield successive DataFrame chunks - process datasets larger than RAM.

Source code in src/cds2/io.py 
114
115
116
117
def iter_csv(path: str, chunksize: int = 100_000, **kwargs: Any) -> Any:
    """Yield successive DataFrame chunks - process datasets larger than RAM."""
    reader = pd.read_csv(path, chunksize=chunksize, **kwargs)
    yield from reader