Skip to content

Commit 698dce3

Browse files
philsvphilsv
committed
Initial commit
1 parent e7c5a32 commit 698dce3

File tree

13 files changed

+453
-1
lines changed

13 files changed

+453
-1
lines changed

‎.github/workflows/pre-commit.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
name: pre-commit
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches: [ main, master ]
7+
8+
jobs:
9+
pre-commit:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v3
13+
- uses: actions/setup-python@v3
14+
- uses: pre-commit/action@v3.0.0

‎.github/workflows/publish.yml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Publish
2+
3+
on:
4+
release:
5+
types: [created]
6+
7+
jobs:
8+
deploy:
9+
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- uses: actions/checkout@v3
14+
- name: Set up Python
15+
uses: actions/setup-python@v3
16+
with:
17+
python-version: '3.8'
18+
- name: Install dependencies
19+
run: |
20+
python -m pip install --upgrade pip
21+
pip install setuptools wheel twine
22+
pip install -r requirements.txt
23+
- name: Build and Publish package
24+
env:
25+
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
26+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
27+
run: |
28+
python setup.py build_ext --inplace
29+
python setup.py sdist bdist_wheel
30+
twine upload dist/*

‎.gitignore

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# C extensions
7+
*.so
8+
9+
# Distribution / packaging
10+
.Python
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
wheels/
23+
share/python-wheels/
24+
*.egg-info/
25+
.installed.cfg
26+
*.egg
27+
MANIFEST
28+
29+
# PyInstaller
30+
# Usually these files are written by a python script from a template
31+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
32+
*.manifest
33+
*.spec
34+
35+
# Installer logs
36+
pip-log.txt
37+
pip-delete-this-directory.txt
38+
39+
# Unit test / coverage reports
40+
htmlcov/
41+
.tox/
42+
.nox/
43+
.coverage
44+
.coverage.*
45+
.cache
46+
nosetests.xml
47+
coverage.xml
48+
*.cover
49+
*.py,cover
50+
.hypothesis/
51+
.pytest_cache/
52+
cover/
53+
54+
# Translations
55+
*.mo
56+
*.pot
57+
58+
# Django stuff:
59+
*.log
60+
local_settings.py
61+
db.sqlite3
62+
db.sqlite3-journal
63+
64+
# Flask stuff:
65+
instance/
66+
.webassets-cache
67+
68+
# Scrapy stuff:
69+
.scrapy
70+
71+
# Sphinx documentation
72+
docs/_build/
73+
74+
# PyBuilder
75+
.pybuilder/
76+
target/
77+
78+
# Jupyter Notebook
79+
.ipynb_checkpoints
80+
81+
# IPython
82+
profile_default/
83+
ipython_config.py
84+
85+
# pyenv
86+
# For a library or package, you might want to ignore these files since the code is
87+
# intended to run in multiple environments; otherwise, check them in:
88+
# .python-version
89+
90+
# pipenv
91+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
93+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
94+
# install all needed dependencies.
95+
#Pipfile.lock
96+
97+
# poetry
98+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99+
# This is especially recommended for binary packages to ensure reproducibility, and is more
100+
# commonly ignored for libraries.
101+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102+
#poetry.lock
103+
104+
# pdm
105+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106+
#pdm.lock
107+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108+
# in version control.
109+
# https://pdm.fming.dev/#use-with-ide
110+
.pdm.toml
111+
112+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113+
__pypackages__/
114+
115+
# Celery stuff
116+
celerybeat-schedule
117+
celerybeat.pid
118+
119+
# SageMath parsed files
120+
*.sage.py
121+
122+
# Environments
123+
.env
124+
.venv
125+
env/
126+
venv/
127+
ENV/
128+
env.bak/
129+
venv.bak/
130+
131+
# Spyder project settings
132+
.spyderproject
133+
.spyproject
134+
135+
# Rope project settings
136+
.ropeproject
137+
138+
# mkdocs documentation
139+
/site
140+
141+
# mypy
142+
.mypy_cache/
143+
.dmypy.json
144+
dmypy.json
145+
146+
# Pyre type checker
147+
.pyre/
148+
149+
# pytype static type analyzer
150+
.pytype/
151+
152+
# Cython debug symbols
153+
cython_debug/
154+
155+
# PyCharm
156+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158+
# and can be added to the global gitignore or merged into this file. For a more nuclear
159+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
160+
#.idea/
161+
162+
# VSCode
163+
.vscode/

‎.pre-commit-config.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
repos:
2+
- repo: https://github.com/pre-commit/pre-commit-hooks
3+
rev: v4.4.0
4+
hooks:
5+
- id: check-docstring-first
6+
- id: check-yaml
7+
8+
- repo: https://github.com/PyCQA/isort
9+
rev: 5.12.0
10+
hooks:
11+
- id: isort

‎LICENSE

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2024 philsv
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎README.md

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,51 @@
1-
# bojpy
1+
# bojpy
2+
3+
[![PyPI version](https://d25lcipzij17d.cloudfront.net/badge.svg?id=py&r=r&ts=1683906897&type=6e&v=0.0.1&x2=0)](https://badge.fury.io/py/bojpy)
4+
[![License: MIT](https://img.shields.io/badge/License-MIT-red.svg)](https://github.com/philsv/bojpy/blob/main/LICENSE)
5+
[![Weekly Downloads](https://static.pepy.tech/personalized-badge/bojpy?period=week&units=international_system&left_color=grey&right_color=blue&left_text=downloads/week)](https://pepy.tech/project/bojpy)
6+
[![Monthly Downloads](https://static.pepy.tech/personalized-badge/bojpy?period=month&units=international_system&left_color=grey&right_color=blue&left_text=downloads/month)](https://pepy.tech/project/bojpy)
7+
[![Downloads](https://static.pepy.tech/personalized-badge/bojpy?period=total&units=international_system&left_color=grey&right_color=blue&left_text=downloads)](https://pepy.tech/project/bojpy)
8+
9+
bojpy is a Python package that provides a simple interface to the [BOJ Time-Series Data Search](https://www.stat-search.boj.or.jp/index_en.html).
10+
11+
## Installation
12+
13+
```ini
14+
pip install bojpy
15+
```
16+
17+
## Requirements
18+
19+
* beautifulsoup4
20+
* pandas
21+
* requests
22+
23+
## How to use
24+
25+
```python
26+
from bojpy import boj
27+
28+
# By data series id
29+
df = boj.get_data_series(series="BS01'MABJMTA")
30+
31+
# By Time-series data html url
32+
url = "https://www.stat-search.boj.or.jp/ssi/html/nme_R020MM.3576038.20240826070325.02.html"
33+
df = boj.get_data_html(url)
34+
```
35+
36+
## Output Example
37+
38+
```ini
39+
Date BS01'MABJMTA Bank of Japan Accounts/Assets/Total(Assets, or Liabilities and Net Assets)(s)
40+
41+
2024-07-01 7617141.0
42+
2024-06-01 7536709.0
43+
2024-05-01 7610851.0
44+
2024-04-01 7583199.0
45+
2024-03-01 7564231.0
46+
... ...
47+
```
48+
49+
## Disclaimer
50+
51+
This package is nor endorsed by nor affiliated with the [Bank of Japan](https://www.boj.or.jp/en/). Please make sure to not abuse the BOJ servers by sending unnecessary requests.

‎bojpy/boj.py

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import warnings
2+
from urllib.parse import urlencode
3+
4+
import numpy as np
5+
import pandas as pd
6+
import requests
7+
from bs4 import BeautifulSoup
8+
9+
warnings.filterwarnings("ignore", category=UserWarning)
10+
11+
12+
def get_data_series(
13+
series: str,
14+
skiprows: int = 0,
15+
) -> pd.DataFrame:
16+
"""
17+
Returns data series from the Bank of Japan (BOJ) Time-Series Data Search.
18+
19+
Example:
20+
>>> get_data_series(series="BS01'MABJMTA")
21+
"""
22+
base_url = "https://www.stat-search.boj.or.jp/ssi/"
23+
search_path = "cgi-bin/famecgi2?cgi=%24nme_r030_en&chkfrq=MM&rdoheader=SIMPLE&rdodelimitar=COMMA&hdnYyyyFrom=&hdnYyyyTo=&sw_freq=NONE&sw_yearend=NONE&sw_observed=NONE&"
24+
series_encoded = urlencode({"hdncode": series})
25+
url = f"{base_url}{search_path}{series_encoded}"
26+
27+
response = requests.get(url)
28+
response.raise_for_status()
29+
30+
page_content = response.content
31+
soup = BeautifulSoup(page_content, "lxml")
32+
nodes = soup.select("a[href*=csv]")
33+
34+
if not nodes:
35+
raise ValueError(f"Could not find .csv file in {url}")
36+
37+
url = f"https://www.stat-search.boj.or.jp/{nodes[0]['href']}"
38+
df = pd.read_csv(url, skiprows=skiprows)
39+
40+
first_row = df.iloc[0]
41+
new_columns = df.columns + " " + first_row
42+
df.columns = new_columns
43+
df = df.drop(index=0)
44+
45+
df[df.columns[0]] = pd.to_datetime(df[df.columns[0]])
46+
df = df.replace({"ND": np.nan}, regex=True)
47+
48+
df = df.rename(columns={df.columns[0]: ""})
49+
df = df.rename_axis("Date", axis=1)
50+
df = df.set_index(df.columns[0])
51+
df = df.astype(float)
52+
53+
if df.index.is_monotonic_increasing:
54+
df = df.sort_index(ascending=False)
55+
56+
df = df.dropna()
57+
return df
58+
59+
60+
def get_data_html(
61+
url: str,
62+
skiprows: int = 0,
63+
) -> pd.DataFrame:
64+
"""
65+
Returns the HTML content as a DataFrame of a given Time-series data URL.
66+
67+
Example:
68+
>>> get_data_html(url="https://www.stat-search.boj.or.jp/ssi/html/nme_R000.3576779.20240826071135.02.html")
69+
"""
70+
df = pd.read_html(url, skiprows=skiprows)[0]
71+
df.columns = df.iloc[0] # type: ignore
72+
df = df.drop(index=0)
73+
74+
df[df.columns[0]] = pd.to_datetime(df[df.columns[0]])
75+
df = df.replace({"ND": np.nan}, regex=True)
76+
77+
df = df.rename(columns={df.columns[0]: ""})
78+
df = df.rename_axis("Date", axis=1)
79+
df = df.set_index(df.columns[0])
80+
df = df.astype(float)
81+
82+
if df.index.is_monotonic_increasing:
83+
df = df.sort_index(ascending=False)
84+
85+
df = df.dropna()
86+
return df

‎bojpy/version.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__version__ = "0.0.1"

‎pyproject.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[tool.pytest.ini_options]
2+
pythonpath = [
3+
".", "bojpy", "tests", "src",
4+
]

‎requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
beautifulsoup4>=4.12.3
2+
pandas>=2.2.2
3+
requests>=2.32.3

0 commit comments

Comments
 (0)