CLN: Clean pandas compat

Ensure concat works correctly
Move testing out of compat

Author: bashtage

linearmodels/compat/pandas.py

@@ -1,3 +1,27 @@
+from distutils.version import LooseVersion
+
+import pandas as pd
+
+PD_LT_023 = LooseVersion(pd.__version__) < LooseVersion('0.23')
+
+
+def concat(*args, **kwargs):
+    """
+    Shim around pandas concat that passes sort if allowed
+
+    See pandas.compat
+    """
+    if PD_LT_023 and 'sort' in kwargs:
+        kwargs = kwargs.copy()
+        del kwargs['sort']
+    else:
+        if 'sort' not in kwargs:
+            kwargs = kwargs.copy()
+            kwargs['sort'] = True
+
+    return pd.concat(*args, **kwargs)
+
+
 try:
     from pandas.api.types import (is_numeric_dtype, is_categorical,
                                   is_string_dtype, is_categorical_dtype,
@@ -24,11 +48,6 @@ def is_string_like(obj):
                                     is_categorical, is_categorical_dtype,
                                     is_datetime64_any_dtype, is_string_like)
 
-try:
-    from pandas.testing import assert_frame_equal, assert_series_equal
-except ImportError:
-    from pandas.util.testing import assert_frame_equal, assert_series_equal
-
 __all__ = ['is_string_dtype', 'is_numeric_dtype', 'is_categorical',
            'is_string_like', 'is_categorical_dtype', 'is_datetime64_any_dtype',
-           'assert_frame_equal', 'assert_series_equal']
+           'concat']

linearmodels/iv/data.py

@@ -8,7 +8,7 @@
 
 from linearmodels.compat.pandas import (is_categorical, is_categorical_dtype,
                                         is_numeric_dtype, is_string_dtype,
-                                        is_string_like)
+                                        is_string_like, concat)
 
 dim_err = '{0} has too many dims.  Maximum is 2, actual is {1}'
 type_err = 'Only ndarrays, DataArrays and Series and DataFrames are supported'
@@ -25,7 +25,7 @@ def convert_columns(s, drop_first):
 def expand_categoricals(x, drop_first):
     if x.shape[1] == 0:
         return x
-    return pd.concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
+    return concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
 
 
 class IVData(object):

linearmodels/panel/data.py

@@ -8,7 +8,7 @@
 from linearmodels.compat.pandas import (is_categorical,
                                         is_datetime64_any_dtype,
                                         is_numeric_dtype, is_string_dtype,
-                                        is_string_like)
+                                        is_string_like, concat)
 from linearmodels.utility import ensure_unique_column, panel_to_frame
 
 __all__ = ['PanelData']
@@ -89,7 +89,7 @@ def convert_columns(s, drop_first):
 
 
 def expand_categoricals(x, drop_first):
-    return pd.concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
+    return concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
 
 
 class PanelData(object):

linearmodels/tests/asset_pricing/test_formulas.py

@@ -1,11 +1,11 @@
 import numpy as np
-import pandas as pd
 import pytest
+from pandas.testing import assert_frame_equal
 
 from linearmodels.asset_pricing.model import (LinearFactorModel,
                                               LinearFactorModelGMM,
                                               TradedFactorModel)
-from linearmodels.compat.pandas import assert_frame_equal
+from linearmodels.compat.pandas import concat
 from linearmodels.tests.asset_pricing._utility import generate_data
 
 FORMULA_FACTORS = 'factor_1 + factor_2 + factor_3'
@@ -29,7 +29,7 @@ def non_traded_model(request):
 def data(request):
     premia = np.array([.1, .1, .1])
     out = generate_data(nportfolio=10, output='pandas', alpha=True, premia=premia)
-    out['joined'] = pd.concat([out.factors, out.portfolios], 1)
+    out['joined'] = concat([out.factors, out.portfolios], 1)
     return out
 
 

linearmodels/tests/iv/test_data.py

@@ -12,7 +12,7 @@
 except ImportError:
     MISSING_XARRAY = True
 
-from linearmodels.compat.pandas import assert_frame_equal, assert_series_equal
+from pandas.testing import assert_frame_equal, assert_series_equal
 from linearmodels.iv.data import IVData
 
 

linearmodels/tests/iv/test_formulas.py

@@ -1,9 +1,10 @@
 import numpy as np
-import pandas as pd
 import pytest
 from numpy.testing import assert_allclose, assert_equal
+from pandas import DataFrame, Categorical
+from pandas.testing import assert_frame_equal
 
-from linearmodels.compat.pandas import assert_frame_equal
+from linearmodels.compat.pandas import concat
 from linearmodels.formula import iv_2sls, iv_gmm, iv_gmm_cue, iv_liml
 from linearmodels.iv import IV2SLS, IVGMM, IVGMMCUE, IVLIML
 
@@ -49,7 +50,7 @@ def data():
     y = x @ params + e
     cols = ['y'] + ['x' + str(i) for i in range(1, 6)]
     cols += ['z' + str(i) for i in range(1, 4)]
-    data = pd.DataFrame(np.c_[y, x, z], columns=cols)
+    data = DataFrame(np.c_[y, x, z], columns=cols)
     data['Intercept'] = 1.0
     data['weights'] = np.random.chisquare(10, size=data.shape[0]) / 10
     return data
@@ -164,8 +165,8 @@ def test_categorical(model_and_func):
     y = np.random.randn(1000)
     x1 = np.random.randn(1000)
     d = np.random.randint(0, 4, 1000)
-    d = pd.Categorical(d)
-    data = pd.DataFrame({'y': y, 'x1': x1, 'd': d})
+    d = Categorical(d)
+    data = DataFrame({'y': y, 'x1': x1, 'd': d})
     data['Intercept'] = 1.0
     model, func = model_and_func
     mod = model.from_formula(formula, data)
@@ -199,7 +200,7 @@ def test_formula_function(data, model_and_func):
     dep = data.y
     exog = [data[['Intercept']], sigmoid(data[['x3']]), data[['x4']],
             np.exp(data[['x5']])]
-    exog = pd.concat(exog, 1)
+    exog = concat(exog, 1)
     endog = data[['x1', 'x2']]
     instr = data[['z1', 'z2', 'z3']]
     mod = model(dep, exog, endog, instr)
@@ -220,7 +221,7 @@ def test_predict_formula_function(data, model_and_func):
 
     exog = [data[['Intercept']], sigmoid(data[['x3']]), data[['x4']],
             np.exp(data[['x5']])]
-    exog = pd.concat(exog, 1)
+    exog = concat(exog, 1)
     endog = data[['x1', 'x2']]
     pred = res.predict(exog, endog)
     pred2 = res.predict(data=data)

linearmodels/tests/iv/test_results.py

@@ -1,8 +1,8 @@
-import pandas as pd
 import pytest
 from numpy.testing import assert_allclose
+from pandas import DataFrame
+from pandas.testing import assert_series_equal
 
-from linearmodels.compat.pandas import assert_series_equal
 from linearmodels.iv.data import IVData
 from linearmodels.iv.model import IV2SLS, IVGMM, IVGMMCUE, IVLIML
 from linearmodels.tests.iv._utility import generate_data
@@ -65,12 +65,12 @@ def test_fitted_predict(data, model):
     assert_series_equal(res.idiosyncratic, res.resids)
     y = mod.dependent.pandas
     expected = y.values - res.resids.values[:, None]
-    expected = pd.DataFrame(expected, y.index, ['fitted_values'])
+    expected = DataFrame(expected, y.index, ['fitted_values'])
     assert_frame_similar(expected, res.fitted_values)
     assert_allclose(expected, res.fitted_values)
     pred = res.predict()
     nobs = res.resids.shape[0]
-    assert isinstance(pred, pd.DataFrame)
+    assert isinstance(pred, DataFrame)
     assert pred.shape == (nobs, 1)
     pred = res.predict(idiosyncratic=True, missing=True)
     nobs = IVData(data.dep).pandas.shape[0]

linearmodels/tests/panel/_utility.py

@@ -1,7 +1,8 @@
 import numpy as np
-import pandas as pd
 from numpy.random import standard_normal
 from numpy.testing import assert_allclose
+from pandas import DataFrame, Categorical, get_dummies, date_range
+from pandas.testing import assert_frame_equal, assert_series_equal
 
 from linearmodels.compat.numpy import lstsq
 from linearmodels.panel.data import PanelData
@@ -13,38 +14,37 @@
     MISSING_XARRAY = False
 except ImportError:
     MISSING_XARRAY = True
-from linearmodels.compat.pandas import assert_frame_equal, assert_series_equal
 from linearmodels.utility import AttrDict
 
 datatypes = ['numpy', 'pandas']
 if not MISSING_XARRAY:
     datatypes += ['xarray']
 
 
-def lsdv(y: pd.DataFrame, x: pd.DataFrame, has_const=False, entity=False, time=False,
+def lsdv(y: DataFrame, x: DataFrame, has_const=False, entity=False, time=False,
          general=None):
     nvar = x.shape[1]
     temp = x.reset_index()
     cat_index = temp.index
     if entity:
-        cat = pd.Categorical(temp.iloc[:, 0])
+        cat = Categorical(temp.iloc[:, 0])
         cat.index = cat_index
-        dummies = pd.get_dummies(cat, drop_first=has_const)
-        x = pd.DataFrame(np.c_[x.values, dummies.values.astype(np.float64)],
+        dummies = get_dummies(cat, drop_first=has_const)
+        x = DataFrame(np.c_[x.values, dummies.values.astype(np.float64)],
                          index=x.index,
                          columns=list(x.columns) + list(dummies.columns))
     if time:
-        cat = pd.Categorical(temp.iloc[:, 1])
+        cat = Categorical(temp.iloc[:, 1])
         cat.index = cat_index
-        dummies = pd.get_dummies(cat, drop_first=(has_const or entity))
-        x = pd.DataFrame(np.c_[x.values, dummies.values.astype(np.float64)],
+        dummies = get_dummies(cat, drop_first=(has_const or entity))
+        x = DataFrame(np.c_[x.values, dummies.values.astype(np.float64)],
                          index=x.index,
                          columns=list(x.columns) + list(dummies.columns))
     if general is not None:
-        cat = pd.Categorical(general)
+        cat = Categorical(general)
         cat.index = cat_index
-        dummies = pd.get_dummies(cat, drop_first=(has_const or entity or time))
-        x = pd.DataFrame(np.c_[x.values, dummies.values.astype(np.float64)],
+        dummies = get_dummies(cat, drop_first=(has_const or entity or time))
+        x = DataFrame(np.c_[x.values, dummies.values.astype(np.float64)],
                          index=x.index,
                          columns=list(x.columns) + list(dummies.columns))
     w = np.ones_like(y)
@@ -92,9 +92,9 @@ def generate_data(missing, datatype, const=False, ntk=(971, 7, 5), other_effects
 
     if datatype in ('pandas', 'xarray'):
         entities = ['firm' + str(i) for i in range(n)]
-        time = pd.date_range('1-1-1900', periods=t, freq='A-DEC')
+        time = date_range('1-1-1900', periods=t, freq='A-DEC')
         var_names = ['x' + str(i) for i in range(k)]
-        # y = pd.DataFrame(y, index=time, columns=entities)
+        # y = DataFrame(y, index=time, columns=entities)
         y = panel_to_frame(y[None], items=['y'], major_axis=time, minor_axis=entities, swap=True)
         w = panel_to_frame(w[None], items=['w'], major_axis=time, minor_axis=entities, swap=True)
         w = w.reindex(y.index)

linearmodels/tests/panel/results/generate-panel-data.py

@@ -1,6 +1,7 @@
 import numpy as np
 import pandas as pd
 
+from linearmodels.compat.pandas import concat
 from linearmodels.panel.data import PanelData
 from linearmodels.utility import panel_to_frame
 
@@ -30,7 +31,7 @@
 y = PanelData(y)
 w = PanelData(w)
 
-z = pd.concat([x.dataframe, y.dataframe, w.dataframe], 1)
+z = concat([x.dataframe, y.dataframe, w.dataframe], 1)
 final_index = pd.MultiIndex.from_product([minor, major])
 final_index.levels[0].name = 'firm'
 z = z.reindex(final_index)