From d826c5385880be2fa6f414a7a567fd1bd6cb7116 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Sun, 27 Jul 2025 17:53:04 +0600 Subject: [PATCH 1/7] feat(plugins): add plugin system for providers with template engine Implement a comprehensive plugin system for PyneCore that allows creating and managing provider plugins. The system includes: - Template engine for generating new provider plugins - Plugin manager for discovery and loading - CLI commands for plugin management - Complete provider plugin template with tests and documentation - Integration with existing provider system The changes enable users to create custom data providers and extend PyneCore's capabilities without modifying core code. --- pytest.ini | 3 + src/pynecore/cli/commands/__init__.py | 4 +- src/pynecore/cli/commands/data.py | 15 +- src/pynecore/cli/commands/plugins.py | 257 +++++++++++++ src/pynecore/cli/plugin_manager.py | 355 ++++++++++++++++++ src/pynecore/cli/template_engine.py | 89 +++++ .../cli/templates/plugins/provider/README.md | 128 +++++++ .../templates/plugins/provider/providers.toml | 55 +++ .../templates/plugins/provider/pyproject.toml | 82 ++++ .../__init__.py | 9 + .../provider.py | 326 ++++++++++++++++ .../plugins/provider/tests/__init__.py | 1 + .../plugins/provider/tests/test_provider.py | 245 ++++++++++++ src/pynecore/providers/__init__.py | 85 ++++- 14 files changed, 1641 insertions(+), 13 deletions(-) create mode 100644 src/pynecore/cli/commands/plugins.py create mode 100644 src/pynecore/cli/plugin_manager.py create mode 100644 src/pynecore/cli/template_engine.py create mode 100644 src/pynecore/cli/templates/plugins/provider/README.md create mode 100644 src/pynecore/cli/templates/plugins/provider/providers.toml create mode 100644 src/pynecore/cli/templates/plugins/provider/pyproject.toml create mode 100644 src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/__init__.py create mode 100644 src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/provider.py create mode 100644 src/pynecore/cli/templates/plugins/provider/tests/__init__.py create mode 100644 src/pynecore/cli/templates/plugins/provider/tests/test_provider.py diff --git a/pytest.ini b/pytest.ini index da48643..3442bba 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,9 @@ [pytest] python_functions = __test_*__ +# Exclude template test files from discovery +norecursedirs = src/pynecore/cli/templates + log_cli = true log_cli_level = DEBUG log_cli_format = %(asctime)s %(levelname)6s %(module_func_line)30s - %(message)s diff --git a/src/pynecore/cli/commands/__init__.py b/src/pynecore/cli/commands/__init__.py index 4534e5f..917f2fc 100644 --- a/src/pynecore/cli/commands/__init__.py +++ b/src/pynecore/cli/commands/__init__.py @@ -9,9 +9,9 @@ from ...providers import available_providers # Import commands -from . import run, data, compile, benchmark +from . import run, data, compile, benchmark, plugins -__all__ = ['run', 'data', 'compile', 'benchmark'] +__all__ = ['run', 'data', 'compile', 'benchmark', 'plugins'] @app.callback() diff --git a/src/pynecore/cli/commands/data.py b/src/pynecore/cli/commands/data.py index 34601fb..7902961 100644 --- a/src/pynecore/cli/commands/data.py +++ b/src/pynecore/cli/commands/data.py @@ -11,7 +11,7 @@ TimeElapsedColumn, TimeRemainingColumn) from ..app import app, app_state -from ...providers import available_providers +from ...providers import get_available_providers, get_provider_class from ...providers.provider import Provider from ...utils.rich.date_column import DateColumn @@ -22,8 +22,9 @@ app_data = Typer(help="OHLCV related commands") app.add_typer(app_data, name="data") -# Create an enum from it -AvailableProvidersEnum = Enum('Provider', {name.upper(): name.lower() for name in available_providers}) +# Create an enum from available providers (built-in + plugins) +_available_providers = get_available_providers() +AvailableProvidersEnum = Enum('Provider', {name.upper(): name.lower() for name in _available_providers}) # Available intervals (The same fmt as described in timeframe.period) TimeframeEnum = Enum('Timeframe', {name: name for name in ('1', '5', '15', '30', '60', '240', '1D', '1W')}) @@ -83,9 +84,11 @@ def download( """ Download historical OHLCV data """ - # Import provider module from - provider_module = __import__(f"pynecore.providers.{provider.value}", fromlist=['']) - provider_class = getattr(provider_module, [p for p in dir(provider_module) if p.endswith('Provider')][0]) + # Get provider class using the new discovery system + provider_class = get_provider_class(provider.value) + if not provider_class: + secho(f"Error: Provider '{provider.value}' not found or failed to load", err=True, fg=colors.RED) + raise Exit(1) try: # If list_symbols is True, we show the available symbols then exit diff --git a/src/pynecore/cli/commands/plugins.py b/src/pynecore/cli/commands/plugins.py new file mode 100644 index 0000000..2c33ae5 --- /dev/null +++ b/src/pynecore/cli/commands/plugins.py @@ -0,0 +1,257 @@ +import typer +from typer import Typer, Argument, Option +from pathlib import Path +from typing import Optional +import shutil +import subprocess +import sys + +from rich.console import Console +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TextColumn +from rich.prompt import Confirm + +from ..app import app, app_state +from ..template_engine import TemplateEngine, get_plugin_templates_dir, get_default_template_variables +from ..plugin_manager import plugin_manager + +app_plugins = Typer(help="Plugin management commands") +app.add_typer(app_plugins, name="plugins") +console = Console() + + +@app_plugins.command() +def list( + plugin_type: Optional[str] = Option(None, help="Filter by plugin type (provider, indicator, strategy)"), + show_errors: bool = Option(False, "--show-errors", help="Show plugins with errors") +): + """List available and installed plugins""" + console.print("[bold blue]Discovering PyneCore plugins...[/bold blue]") + plugin_manager.list_plugins(plugin_type=plugin_type, show_errors=show_errors) + + +@app_plugins.command() +def create( + plugin_name: str = Argument(..., help="Name of the plugin to create"), + plugin_type: str = Option("provider", help="Type of plugin (provider, indicator, etc.)"), + output_dir: Path = Option(Path.cwd(), help="Output directory for plugin"), + force: bool = Option(False, "--force", help="Overwrite existing plugin directory") +): + """Create a new plugin template""" + # Validate plugin type + supported_types = ["provider"] # Can be extended later + if plugin_type not in supported_types: + console.print(f"[red]Error: Unsupported plugin type '{plugin_type}'[/red]") + console.print(f"Supported types: {', '.join(supported_types)}") + raise typer.Exit(1) + + # Validate plugin name + if not plugin_name.replace('-', '').replace('_', '').isalnum(): + console.print(f"[red]Error: Plugin name '{plugin_name}' contains invalid characters[/red]") + console.print("Plugin names should only contain letters, numbers, hyphens, and underscores") + raise typer.Exit(1) + + # Determine output directory + plugin_dir = output_dir / f"pynecore-{plugin_name.lower().replace('_', '-')}-{plugin_type}" + + # Check if directory already exists + if plugin_dir.exists(): + if not force: + if not Confirm.ask(f"Directory '{plugin_dir}' already exists. Overwrite?"): + console.print("[yellow]Plugin creation cancelled[/yellow]") + raise typer.Exit(0) + + # Remove existing directory + shutil.rmtree(plugin_dir) + + # Get template directory + template_dir = get_plugin_templates_dir() / plugin_type + if not template_dir.exists(): + console.print(f"[red]Error: No template found for plugin type '{plugin_type}'[/red]") + raise typer.Exit(1) + + # Initialize template engine + template_engine = TemplateEngine() + template_variables = get_default_template_variables(plugin_name, plugin_type) + template_engine.set_variables(template_variables) + + console.print(f"[bold green]Creating {plugin_type} plugin '{plugin_name}'...[/bold green]") + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task("Generating plugin files...", total=None) + + try: + # Create plugin directory + plugin_dir.mkdir(parents=True, exist_ok=True) + + # Process all template files + _process_template_directory(template_dir, plugin_dir, template_engine, progress, task) + + progress.update(task, description="Plugin created successfully!") + + except Exception as e: + console.print(f"[red]Error creating plugin: {e}[/red]") + # Clean up on error + if plugin_dir.exists(): + shutil.rmtree(plugin_dir) + raise typer.Exit(1) + + # Show success message with next steps + _show_creation_success(plugin_name, plugin_type, plugin_dir) + + +def _process_template_directory(template_dir: Path, output_dir: Path, template_engine: TemplateEngine, progress, task): + """Recursively process template directory""" + # Skip patterns for files/directories that shouldn't be processed + skip_patterns = { + '__pycache__', + '.pyc', + '.pyo', + '.pyd', + '.so', + '.dylib', + '.dll', + '.git', + '.svn', + '.hg', + '.DS_Store', + 'Thumbs.db' + } + + for item in template_dir.rglob("*"): + # Skip if any part of the path contains skip patterns + if any(skip_pattern in str(item) for skip_pattern in skip_patterns): + continue + + # Skip if file extension is in skip patterns + if item.suffix in {'.pyc', '.pyo', '.pyd', '.so', '.dylib', '.dll'}: + continue + + if item.is_file(): + # Calculate relative path from template directory + rel_path = item.relative_to(template_dir) + + # Process path template variables (e.g., {{plugin_name_snake}}_provider) + output_path_str = str(rel_path) + output_path_str = template_engine.render(output_path_str) + output_path = output_dir / output_path_str + + # Ensure output directory exists + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Update progress + progress.update(task, description=f"Processing {rel_path}...") + + try: + # Render template file + template_engine.render_file(item, output_path) + except UnicodeDecodeError: + # If we can't decode the file as text, copy it as binary + shutil.copy2(item, output_path) + + +def _show_creation_success(plugin_name: str, plugin_type: str, plugin_dir: Path): + """Show success message with next steps""" + plugin_name_kebab = plugin_name.lower().replace('_', '-') + + success_text = f""" +[bold green]✓ Plugin created successfully![/bold green] + +[bold]Plugin Details:[/bold] +• Name: {plugin_name} +• Type: {plugin_type} +• Directory: {plugin_dir} +• Package: pynecore-{plugin_name_kebab}-{plugin_type} + +[bold]Next Steps:[/bold] + +1. [cyan]Navigate to the plugin directory:[/cyan] + cd "{plugin_dir}" + +2. [cyan]Customize the implementation:[/cyan] + • Edit src/{plugin_name.lower().replace('-', '_')}_provider/provider.py + • Update configuration in providers.toml + • Modify pyproject.toml with your details + +3. [cyan]Install in development mode:[/cyan] + pip install -e ".[dev]" + +4. [cyan]Test your plugin:[/cyan] + pytest + +5. [cyan]Use with PyneCore:[/cyan] + pyne data download --provider {plugin_name.lower().replace('-', '_')} --list-symbols + +[bold]Documentation:[/bold] +See README.md in the plugin directory for detailed implementation guidance. + """ + + panel = Panel( + success_text.strip(), + title=f"Plugin '{plugin_name}' Created", + border_style="green" + ) + + console.print(panel) + + +@app_plugins.command() +def install( + plugin_name: str = Argument(..., help="Name of the plugin to install from PyPI"), + upgrade: bool = Option(False, "--upgrade", "-U", help="Upgrade if already installed") +): + """Install a plugin from PyPI""" + # Construct expected package name + if not plugin_name.startswith("pynecore-"): + package_name = f"pynecore-{plugin_name}" + else: + package_name = plugin_name + + console.print(f"[bold blue]Installing plugin: {package_name}[/bold blue]") + + # Prepare pip command + cmd = [sys.executable, "-m", "pip", "install"] + if upgrade: + cmd.append("--upgrade") + cmd.append(package_name) + + try: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + task = progress.add_task(f"Installing {package_name}...", total=None) + + # Run pip install + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + progress.update(task, description="Installation completed!") + console.print(f"[green]✓ Successfully installed {package_name}[/green]") + + # Try to discover the new plugin + console.print("[blue]Discovering new plugin...[/blue]") + plugin_manager.discover_plugins() + + else: + console.print(f"[red]✗ Failed to install {package_name}[/red]") + console.print(f"[red]Error: {result.stderr}[/red]") + raise typer.Exit(1) + + except Exception as e: + console.print(f"[red]Error during installation: {e}[/red]") + raise typer.Exit(1) + + +@app_plugins.command() +def info( + plugin_name: str = Argument(..., help="Name of the plugin to show info for") +): + """Show plugin information and status""" + console.print(f"[bold blue]Getting information for plugin: {plugin_name}[/bold blue]") + plugin_manager.show_plugin_info(plugin_name) \ No newline at end of file diff --git a/src/pynecore/cli/plugin_manager.py b/src/pynecore/cli/plugin_manager.py new file mode 100644 index 0000000..ea698c9 --- /dev/null +++ b/src/pynecore/cli/plugin_manager.py @@ -0,0 +1,355 @@ +"""Plugin Manager for PyneCore + +This module handles plugin discovery, registration, and management using entry points. +""" + +from typing import Dict, List, Any, Optional, Type +from pathlib import Path +import importlib.metadata +import importlib.util +import sys +from dataclasses import dataclass + +from rich.console import Console +from rich.table import Table +from rich.panel import Panel + + +@dataclass +class PluginInfo: + """Information about a discovered plugin""" + name: str + version: str + description: str + entry_point: str + plugin_type: str + module_name: str + class_name: str + installed: bool = False + loaded: bool = False + error: Optional[str] = None + + +class PluginManager: + """Manages PyneCore plugins using entry points""" + + def __init__(self): + self.console = Console() + self._discovered_plugins: Dict[str, PluginInfo] = {} + self._loaded_plugins: Dict[str, Any] = {} + + # Entry point groups for different plugin types + self.entry_point_groups = { + "provider": "pynecore.providers", + "indicator": "pynecore.indicators", + "strategy": "pynecore.strategies", + } + + def discover_plugins(self, plugin_type: Optional[str] = None) -> Dict[str, PluginInfo]: + """Discover plugins using entry points and built-in providers + + Args: + plugin_type: Optional plugin type to filter by (provider, indicator, strategy) + + Returns: + Dictionary of discovered plugins + """ + discovered = {} + + # Add built-in providers if we're looking for providers or all types + if plugin_type is None or plugin_type == "provider": + discovered.update(self._discover_builtin_providers()) + + # Determine which entry point groups to check + if plugin_type and plugin_type in self.entry_point_groups: + groups_to_check = {plugin_type: self.entry_point_groups[plugin_type]} + else: + groups_to_check = self.entry_point_groups + + for ptype, group_name in groups_to_check.items(): + try: + # Get entry points for this group + entry_points = importlib.metadata.entry_points().select(group=group_name) + + for entry_point in entry_points: + try: + # Get distribution info + dist = entry_point.dist + + plugin_info = PluginInfo( + name=entry_point.name, + version=dist.version if dist else "unknown", + description=self._get_package_description(dist), + entry_point=f"{entry_point.module}:{entry_point.attr}", + plugin_type=ptype, + module_name=entry_point.module, + class_name=entry_point.attr, + installed=True + ) + + discovered[entry_point.name] = plugin_info + + except Exception as e: + # Create error entry for failed plugin + error_info = PluginInfo( + name=entry_point.name, + version="unknown", + description="Failed to load plugin info", + entry_point=f"{entry_point.module}:{entry_point.attr}", + plugin_type=ptype, + module_name=entry_point.module, + class_name=entry_point.attr, + installed=True, + error=str(e) + ) + discovered[entry_point.name] = error_info + + except Exception as e: + self.console.print(f"[red]Error discovering plugins for group {group_name}: {e}[/red]") + + self._discovered_plugins.update(discovered) + return discovered + + def _discover_builtin_providers(self) -> Dict[str, PluginInfo]: + """Discover built-in providers + + Returns: + Dictionary of built-in provider info + """ + builtin_plugins = {} + + try: + # Import built-in providers info + from ..providers import _builtin_providers + + for name, provider_class in _builtin_providers.items(): + # Get version from the main package + try: + import pynecore + version = getattr(pynecore, '__version__', 'unknown') + except: + version = 'unknown' + + # Get description from provider class docstring + description = "Built-in provider" + if provider_class.__doc__: + # Get first line of docstring + description = provider_class.__doc__.strip().split('\n')[0] + + plugin_info = PluginInfo( + name=name, + version=version, + description=description, + entry_point=f"{provider_class.__module__}:{provider_class.__name__}", + plugin_type="provider", + module_name=provider_class.__module__, + class_name=provider_class.__name__, + installed=True, + loaded=True # Built-in providers are always "loaded" + ) + + builtin_plugins[name] = plugin_info + + except ImportError: + # Built-in providers not available + pass + + return builtin_plugins + + def _get_package_description(self, dist) -> str: + """Get package description from distribution metadata""" + if not dist: + return "No description available" + + try: + metadata = dist.metadata + return metadata.get("Summary", metadata.get("Description", "No description available")) + except Exception: + return "No description available" + + def load_plugin(self, plugin_name: str) -> Optional[Any]: + """Load a specific plugin by name + + Args: + plugin_name: Name of the plugin to load + + Returns: + Loaded plugin class or None if failed + """ + if plugin_name in self._loaded_plugins: + return self._loaded_plugins[plugin_name] + + # Discover plugins if not already done + if plugin_name not in self._discovered_plugins: + self.discover_plugins() + + if plugin_name not in self._discovered_plugins: + self.console.print(f"[red]Plugin '{plugin_name}' not found[/red]") + return None + + plugin_info = self._discovered_plugins[plugin_name] + + if plugin_info.error: + self.console.print(f"[red]Plugin '{plugin_name}' has errors: {plugin_info.error}[/red]") + return None + + try: + # Load the module + module = importlib.import_module(plugin_info.module_name) + + # Get the plugin class + plugin_class = getattr(module, plugin_info.class_name) + + # Cache the loaded plugin + self._loaded_plugins[plugin_name] = plugin_class + plugin_info.loaded = True + + return plugin_class + + except Exception as e: + error_msg = f"Failed to load plugin: {e}" + plugin_info.error = error_msg + self.console.print(f"[red]{error_msg}[/red]") + return None + + def get_available_providers(self) -> List[str]: + """Get list of available provider plugin names + + Returns: + List of provider plugin names + """ + providers = self.discover_plugins("provider") + return [name for name, info in providers.items() if not info.error] + + def get_plugin_info(self, plugin_name: str) -> Optional[PluginInfo]: + """Get information about a specific plugin + + Args: + plugin_name: Name of the plugin + + Returns: + PluginInfo object or None if not found + """ + if plugin_name not in self._discovered_plugins: + self.discover_plugins() + + return self._discovered_plugins.get(plugin_name) + + def list_plugins(self, plugin_type: Optional[str] = None, show_errors: bool = False) -> None: + """Display a formatted list of available plugins + + Args: + plugin_type: Optional plugin type to filter by + show_errors: Whether to show plugins with errors + """ + plugins = self.discover_plugins(plugin_type) + + if not plugins: + self.console.print("[yellow]No plugins found[/yellow]") + return + + # Display status meanings + self.console.print("[bold blue]Status Meanings:[/bold blue]") + self.console.print(" [green]✓ Loaded[/green] - Plugin is loaded and ready to use") + self.console.print(" [yellow]○ Available[/yellow] - Plugin is installed and will load on demand") + if show_errors: + self.console.print(" [red]✗ Error[/red] - Plugin has errors and cannot be loaded") + self.console.print() + + # Filter out error plugins if requested + if not show_errors: + plugins = {name: info for name, info in plugins.items() if not info.error} + + # Group by plugin type + by_type = {} + for name, info in plugins.items(): + if info.plugin_type not in by_type: + by_type[info.plugin_type] = [] + by_type[info.plugin_type].append((name, info)) + + for ptype, plugin_list in by_type.items(): + # Create table for this plugin type + table = Table(title=f"{ptype.title()} Plugins") + table.add_column("Name", style="cyan") + table.add_column("Version", style="green") + table.add_column("Description", style="white") + table.add_column("Status", style="yellow") + + for name, info in sorted(plugin_list): + status = "✓ Loaded" if info.loaded else "○ Available" + if info.error: + status = "✗ Error" + + description = info.description + if info.error and show_errors: + description += f" (Error: {info.error})" + + table.add_row(name, info.version, description, status) + + self.console.print(table) + self.console.print() + + def show_plugin_info(self, plugin_name: str) -> None: + """Display detailed information about a specific plugin + + Args: + plugin_name: Name of the plugin to show info for + """ + plugin_info = self.get_plugin_info(plugin_name) + + if not plugin_info: + self.console.print(f"[red]Plugin '{plugin_name}' not found[/red]") + return + + # Create info panel + info_text = f""" +[bold]Name:[/bold] {plugin_info.name} +[bold]Version:[/bold] {plugin_info.version} +[bold]Type:[/bold] {plugin_info.plugin_type} +[bold]Entry Point:[/bold] {plugin_info.entry_point} +[bold]Description:[/bold] {plugin_info.description} +[bold]Installed:[/bold] {'Yes' if plugin_info.installed else 'No'} +[bold]Loaded:[/bold] {'Yes' if plugin_info.loaded else 'No'} + """ + + if plugin_info.error: + info_text += f"\n[bold red]Error:[/bold red] {plugin_info.error}" + + panel = Panel( + info_text.strip(), + title=f"Plugin Information: {plugin_name}", + border_style="blue" + ) + + self.console.print(panel) + + def validate_plugin_structure(self, plugin_path: Path) -> bool: + """Validate that a plugin directory has the correct structure + + Args: + plugin_path: Path to the plugin directory + + Returns: + True if structure is valid, False otherwise + """ + required_files = [ + "pyproject.toml", + "README.md", + ] + + for file_name in required_files: + if not (plugin_path / file_name).exists(): + self.console.print(f"[red]Missing required file: {file_name}[/red]") + return False + + # Check for src directory structure + src_dir = plugin_path / "src" + if not src_dir.exists(): + self.console.print(f"[red]Missing src directory[/red]") + return False + + return True + + +# Global plugin manager instance +plugin_manager = PluginManager() \ No newline at end of file diff --git a/src/pynecore/cli/template_engine.py b/src/pynecore/cli/template_engine.py new file mode 100644 index 0000000..34f8019 --- /dev/null +++ b/src/pynecore/cli/template_engine.py @@ -0,0 +1,89 @@ +from pathlib import Path +from typing import Dict, Any +import re +from datetime import datetime + + +class TemplateEngine: + """Simple template engine for plugin generation""" + + def __init__(self): + self.variables = {} + + def set_variables(self, variables: Dict[str, Any]): + """Set template variables""" + self.variables.update(variables) + + def render(self, template_content: str) -> str: + """Render template with variables""" + result = template_content + + # Replace variables in format {{variable_name}} + for key, value in self.variables.items(): + pattern = f"{{{{\s*{key}\s*}}}}" + result = re.sub(pattern, str(value), result) + + return result + + def render_file(self, template_path: Path, output_path: Path): + """Render template file to output file""" + with open(template_path, 'r', encoding='utf-8') as f: + template_content = f.read() + + rendered_content = self.render(template_content) + + # Ensure output directory exists + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, 'w', encoding='utf-8') as f: + f.write(rendered_content) + + +def get_plugin_templates_dir() -> Path: + """Get the plugin templates directory""" + return Path(__file__).parent / "templates" / "plugins" + + +def get_default_template_variables(plugin_name: str, plugin_type: str) -> Dict[str, Any]: + """Get default template variables for plugin generation""" + # Convert plugin name to various formats + plugin_name_snake = plugin_name.lower().replace('-', '_').replace(' ', '_') + plugin_name_pascal = ''.join(word.capitalize() for word in plugin_name_snake.split('_')) + plugin_name_kebab = plugin_name_snake.replace('_', '-') + + # Get PyneCore version from main pyproject.toml + pynecore_version = _get_pynecore_version() + + return { + 'plugin_name': plugin_name, + 'plugin_name_snake': plugin_name_snake, + 'plugin_name_pascal': plugin_name_pascal, + 'plugin_name_kebab': plugin_name_kebab, + 'plugin_type': plugin_type, + 'current_year': datetime.now().year, + 'current_date': datetime.now().strftime('%Y-%m-%d'), + 'pynecore_version': pynecore_version, + } + + +def _get_pynecore_version() -> str: + """Get PyneCore version from main pyproject.toml""" + try: + # Get the path to the main pyproject.toml (relative to this file) + current_file = Path(__file__) + # Navigate up to the project root: cli/template_engine.py -> cli -> pynecore -> src -> root + project_root = current_file.parent.parent.parent.parent + pyproject_path = project_root / "pyproject.toml" + + if pyproject_path.exists(): + import tomllib + with open(pyproject_path, 'rb') as f: + data = tomllib.load(f) + version = data.get('project', {}).get('version', '0.1.0') + return f">={version}" + else: + # Fallback if pyproject.toml not found + return ">=0.1.0" + except Exception: + # Fallback on any error + return ">=0.1.0" \ No newline at end of file diff --git a/src/pynecore/cli/templates/plugins/provider/README.md b/src/pynecore/cli/templates/plugins/provider/README.md new file mode 100644 index 0000000..68079e3 --- /dev/null +++ b/src/pynecore/cli/templates/plugins/provider/README.md @@ -0,0 +1,128 @@ +# {{plugin_name_pascal}} Provider for PyneCore + +A PyneCore data provider plugin for accessing {{plugin_name}} market data. + +## Installation + +```bash +pip install pynecore-{{plugin_name_kebab}}-provider +``` + +## Configuration + +Add the following configuration to your `providers.toml` file: + +```toml +[{{plugin_name_snake}}] +# Add your provider-specific configuration here +# api_key = "your_api_key_here" +# api_secret = "your_api_secret_here" +# base_url = "https://api.example.com" +``` + +## Usage + +Once installed, the provider will be automatically discovered by PyneCore. You can use it with the CLI: + +```bash +# List available symbols +pyne data download --provider {{plugin_name_snake}} --list-symbols + +# Download OHLCV data +pyne data download --provider {{plugin_name_snake}} --symbol EURUSD --timeframe 1h --time-from 2024-01-01 --time-to 2024-01-31 +``` + +Or programmatically: + +```python +from pathlib import Path +from {{plugin_name_snake}}_provider import {{plugin_name_pascal}}Provider +from datetime import datetime, timezone + +# Initialize provider +provider = {{plugin_name_pascal}}Provider( + symbol="EURUSD", + timeframe="1h", + workdir=Path("./data") +) + +# Download data +data = provider.download_ohlcv( + symbol="EURUSD", + timeframe="1h", + time_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + time_to=datetime(2024, 1, 31, tzinfo=timezone.utc) +) + +print(f"Downloaded {len(data)} candles") +if data: + print(f"First candle: {data[0]}") + print(f"Last candle: {data[-1]}") +``` + +## Development + +### Setup + +```bash +# Clone the repository +git clone https://github.com/yourusername/pynecore-{{plugin_name_kebab}}-provider.git +cd pynecore-{{plugin_name_kebab}}-provider + +# Install in development mode +pip install -e ".[dev]" +``` + +### Testing + +```bash +# Run tests +pytest + +# Run tests with coverage +pytest --cov={{plugin_name_snake}}_provider +``` + +### Code Quality + +```bash +# Format code +black src/ tests/ +isort src/ tests/ + +# Lint code +flake8 src/ tests/ +mypy src/ +``` + +## Implementation Notes + +This provider template includes: + +- ✅ Complete PyneCore Provider interface implementation +- ✅ Proper entry point configuration for auto-discovery +- ✅ Configuration management through `providers.toml` +- ✅ Timeframe conversion utilities +- ✅ Symbol information and trading hours support +- ✅ Progress callback support for data downloads +- ✅ Comprehensive error handling +- ✅ Type hints and documentation +- ✅ Development tools configuration + +### TODO: Customize for Your Provider + +1. **Update API Integration**: Replace placeholder implementations in `provider.py` with actual API calls +2. **Configure Authentication**: Add required API keys/secrets to configuration +3. **Implement Timeframe Mapping**: Update timeframe conversion methods for your provider's format +4. **Add Symbol Mapping**: Implement symbol listing and information retrieval +5. **Handle Rate Limiting**: Add appropriate rate limiting and retry logic +6. **Add Tests**: Create comprehensive test suite for your provider +7. **Update Documentation**: Customize this README with provider-specific information + +## License + +MIT License - see LICENSE file for details. + +## Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. \ No newline at end of file diff --git a/src/pynecore/cli/templates/plugins/provider/providers.toml b/src/pynecore/cli/templates/plugins/provider/providers.toml new file mode 100644 index 0000000..9253da6 --- /dev/null +++ b/src/pynecore/cli/templates/plugins/provider/providers.toml @@ -0,0 +1,55 @@ +# {{plugin_name_pascal}} Provider Configuration +# Add this configuration to your PyneCore providers.toml file + +[{{plugin_name_snake}}] +# Provider-specific configuration +# Uncomment and configure the following settings as needed: + +# API credentials (if required) +# api_key = "your_api_key_here" +# api_secret = "your_api_secret_here" +# api_passphrase = "your_passphrase_here" # Some providers require this + +# API endpoints +# base_url = "https://api.example.com" +# sandbox_url = "https://sandbox-api.example.com" # For testing +# use_sandbox = false # Set to true for testing + +# Rate limiting +# requests_per_second = 10 +# requests_per_minute = 600 + +# Timeouts (in seconds) +# connect_timeout = 10 +# read_timeout = 30 + +# Data settings +# max_candles_per_request = 1000 +# default_timeframe = "1h" + +# Retry settings +# max_retries = 3 +# retry_delay = 1.0 # seconds + +# Logging +# log_level = "INFO" # DEBUG, INFO, WARNING, ERROR +# log_requests = false # Log all API requests + +# Example configuration for common provider types: + +# For REST API providers: +# [{{plugin_name_snake}}] +# api_key = "your_api_key" +# base_url = "https://api.example.com/v1" +# requests_per_second = 10 + +# For WebSocket providers: +# [{{plugin_name_snake}}] +# api_key = "your_api_key" +# ws_url = "wss://ws.example.com/v1" +# reconnect_interval = 30 + +# For file-based providers: +# [{{plugin_name_snake}}] +# data_directory = "/path/to/data" +# file_format = "csv" # csv, parquet, json \ No newline at end of file diff --git a/src/pynecore/cli/templates/plugins/provider/pyproject.toml b/src/pynecore/cli/templates/plugins/provider/pyproject.toml new file mode 100644 index 0000000..7690704 --- /dev/null +++ b/src/pynecore/cli/templates/plugins/provider/pyproject.toml @@ -0,0 +1,82 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pynecore-{{plugin_name_kebab}}-provider" +version = "0.1.0" +description = "{{plugin_name_pascal}} data provider plugin for PyneCore" +authors = [ + {name = "Your Name", email = "your.email@example.com"}, +] +license = {text = "MIT"} +readme = "README.md" +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Office/Business :: Financial :: Investment", + "Topic :: Scientific/Engineering :: Information Analysis", +] +keywords = ["trading", "finance", "data", "provider", "pynecore"] + +dependencies = [ + "pynesys-pynecore{{pynecore_version}}", + # Add your provider-specific dependencies here + # "requests>=2.28.0", + # "websocket-client>=1.4.0", + # Note: Use only raw Python and PyneCore methods, avoid pandas/numpy +] + +[project.optional-dependencies] +dev = [ + "pytest>=6.0", + "pytest-cov>=2.0", + "black>=21.0", + "isort>=5.0", + "flake8>=3.8", + "mypy>=0.800", +] + +[project.urls] +Homepage = "https://github.com/yourusername/pynecore-{{plugin_name_kebab}}-provider" +Repository = "https://github.com/yourusername/pynecore-{{plugin_name_kebab}}-provider" +Issues = "https://github.com/yourusername/pynecore-{{plugin_name_kebab}}-provider/issues" +Documentation = "https://github.com/yourusername/pynecore-{{plugin_name_kebab}}-provider#readme" + +# Entry point for PyneCore plugin discovery +[project.entry-points."pynecore.providers"] +{{plugin_name_snake}} = "{{plugin_name_snake}}_provider:{{plugin_name_pascal}}Provider" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +"*" = ["*.toml", "*.yaml", "*.yml", "*.json"] + +[tool.black] +line-length = 88 +target-version = ['py38'] +include = '\.pyi?$' + +[tool.isort] +profile = "black" +line_length = 88 + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra -q --strict-markers" +testpaths = ["tests"] \ No newline at end of file diff --git a/src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/__init__.py b/src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/__init__.py new file mode 100644 index 0000000..b27a2e5 --- /dev/null +++ b/src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/__init__.py @@ -0,0 +1,9 @@ +"""{{plugin_name_pascal}} Provider Plugin for PyneCore + +This module provides data access for {{plugin_name}} through PyneCore's provider interface. +""" + +from .provider import {{plugin_name_pascal}}Provider + +__version__ = "0.1.0" +__all__ = ["{{plugin_name_pascal}}Provider"] \ No newline at end of file diff --git a/src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/provider.py b/src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/provider.py new file mode 100644 index 0000000..1b049a7 --- /dev/null +++ b/src/pynecore/cli/templates/plugins/provider/src/{{plugin_name_snake}}_provider/provider.py @@ -0,0 +1,326 @@ +"""{{plugin_name_pascal}} Provider Implementation + +This module implements the {{plugin_name_pascal}}Provider class that extends PyneCore's Provider base class. +""" + +from typing import List, Dict, Any, Optional, Tuple, Callable +from datetime import datetime, timezone, timedelta +from pathlib import Path + +from pynecore.providers.provider import Provider +from pynecore.types.ohlcv import OHLCV +from pynecore.core.syminfo import SymInfo, SymInfoInterval, SymInfoSession + + +class {{plugin_name_pascal}}Provider(Provider): + """{{plugin_name_pascal}} data provider for PyneCore + + This provider implements data access for {{plugin_name}} markets. + """ + + def __init__(self, *, symbol: str | None = None, timeframe: str | None = None, + ohlv_dir: Path | None = None, config_dir: Path | None = None): + """Initialize {{plugin_name_pascal}} provider + + Args: + symbol: Trading symbol (e.g., 'EURUSD', 'BTCUSD') + timeframe: Timeframe for data (e.g., '1m', '5m', '1h', '1d') + ohlv_dir: Directory to save OHLCV data + config_dir: Directory to read config file from + """ + super().__init__(symbol=symbol, timeframe=timeframe, ohlv_dir=ohlv_dir, config_dir=config_dir) + + # Set provider-specific timezone + self.timezone = timezone.utc # Adjust based on your provider's timezone + + # Define configuration keys required for this provider + self.config_keys = { + '# Settings for {{plugin_name}} provider': '', + # Add required configuration keys here + # "api_key": "your_api_key_here", + # "api_secret": "your_api_secret_here", + # "base_url": "https://api.example.com", + } + + # Initialize provider-specific client/connection + self._client = None + self._initialize_client() + + def _initialize_client(self): + """Initialize the provider's client/connection""" + # TODO: Initialize your provider's client here + # Example: + # self._client = SomeAPIClient( + # api_key=self.config.get("api_key"), + # api_secret=self.config.get("api_secret"), + # base_url=self.config.get("base_url", "https://api.example.com") + # ) + pass + + @classmethod + def to_tradingview_timeframe(cls, timeframe: str) -> str: + """Convert provider timeframe to TradingView format + + Args: + timeframe: Provider's native timeframe format + + Returns: + TradingView compatible timeframe string + """ + # TODO: Implement timeframe conversion from your provider to TradingView + # Example mapping: + timeframe_map = { + "1m": "1", + "5m": "5", + "15m": "15", + "30m": "30", + "1h": "60", + "4h": "240", + "1d": "1D", + "1w": "1W", + "1M": "1M", + } + return timeframe_map.get(timeframe, timeframe) + + @classmethod + def to_exchange_timeframe(cls, timeframe: str) -> str: + """Convert TradingView timeframe to provider format + + Args: + timeframe: TradingView timeframe format + + Returns: + Provider's native timeframe string + """ + # TODO: Implement timeframe conversion from TradingView to your provider + # Example mapping (reverse of above): + timeframe_map = { + "1": "1m", + "5": "5m", + "15": "15m", + "30": "30m", + "60": "1h", + "240": "4h", + "1D": "1d", + "1W": "1w", + "1M": "1M", + } + return timeframe_map.get(timeframe, timeframe) + + def get_list_of_symbols(self, *args, **kwargs) -> list[str]: + """Get list of available symbols from the provider + + Returns: + List of available trading symbols + """ + # TODO: Implement symbol listing from your provider + # Example: + # try: + # response = self._client.get_symbols() + # return [symbol['name'] for symbol in response] + # except Exception as e: + # raise RuntimeError(f"Failed to fetch symbols: {e}") + + # Placeholder implementation + return ["EURUSD", "GBPUSD", "USDJPY", "BTCUSD", "ETHUSD"] + + def update_symbol_info(self) -> SymInfo: + """Update and return symbol information + + Args: + symbol: Trading symbol to get info for + + Returns: + Dictionary containing symbol information + """ + # TODO: Implement symbol info retrieval from your provider + # Example: + # try: + # response = self._client.get_symbol_info(symbol) + # return { + # "symbol": symbol, + # "description": response.get("description", ""), + # "type": response.get("type", "forex"), + # "currency": response.get("currency", "USD"), + # "exchange": response.get("exchange", "{{plugin_name}}"), + # "min_movement": response.get("min_movement", 0.00001), + # "price_scale": response.get("price_scale", 100000), + # "timezone": str(self.timezone), + # "session": "24x7", + # } + # except Exception as e: + # raise RuntimeError(f"Failed to fetch symbol info for {symbol}: {e}") + + # Placeholder implementation + from pynecore.core.syminfo import SymInfo + + return SymInfo( + prefix="{{plugin_name}}", + description=f"{self.symbol or 'UNKNOWN'} from {{plugin_name}}", + ticker=self.symbol or "UNKNOWN", + currency="USD", + period=self.timeframe or "1h", + type="forex" if self.symbol and ("/" in self.symbol or len(self.symbol) == 6) else "crypto", + mintick=0.00001, + pricescale=100000, + pointvalue=1.0, + opening_hours=[], + session_starts=[], + session_ends=[], + timezone=str(self.timezone), + avg_spread=None, + taker_fee=0.001 + ) + + def get_opening_hours_and_sessions(self) -> tuple[list[SymInfoInterval], list[SymInfoSession], list[SymInfoSession]]: + """Get trading hours and session information for symbol + + Returns: + Tuple containing (opening_hours, sessions, session_ends) + """ + # TODO: Implement session info retrieval from your provider + # Example for forex: + # opening_hours = [ + # SymInfoInterval(start="00:00", end="24:00", days=[1, 2, 3, 4, 5]) + # ] + # sessions = [ + # SymInfoSession(name="regular", start="00:00", end="24:00", days=[1, 2, 3, 4, 5]) + # ] + # session_ends = [] + # return opening_hours, sessions, session_ends + + # Placeholder implementation (24/7 trading) + opening_hours = [] + sessions = [] + session_ends = [] + return opening_hours, sessions, session_ends + + def download_ohlcv(self, time_from: datetime, time_to: datetime, + on_progress: Callable[[datetime], None] | None = None): + """Download OHLCV data from the provider + + In the user code you can call `self.save_ohlcv_data()` to save the data into the data file + + Args: + time_from: Start datetime (timezone-aware) + time_to: End datetime (timezone-aware) + on_progress: Optional callback to call on progress + """ + # TODO: Implement OHLCV data download from your provider + # Example: + # try: + # provider_timeframe = self.to_exchange_timeframe(self.timeframe) + # + # # Convert timestamps to provider's expected format + # start_ts = int(time_from.timestamp() * 1000) # milliseconds + # end_ts = int(time_to.timestamp() * 1000) + # + # # Fetch data in chunks if needed + # current_start = start_ts + # + # while current_start < end_ts: + # if on_progress: + # current_time = datetime.fromtimestamp(current_start / 1000, tz=timezone.utc) + # on_progress(current_time) + # + # # Calculate chunk end (e.g., 1000 candles at a time) + # chunk_end = min(current_start + (1000 * timeframe_to_seconds(provider_timeframe) * 1000), end_ts) + # + # # Fetch chunk + # chunk_data = self._client.get_ohlcv( + # symbol=self.symbol, + # timeframe=provider_timeframe, + # start=current_start, + # end=chunk_end + # ) + # + # if not chunk_data: + # break + # + # # Convert to OHLCV objects and save + # ohlcv_data = [] + # for row in chunk_data: + # ohlcv = OHLCV( + # timestamp=int(row[0]), # Unix timestamp in milliseconds + # open=float(row[1]), + # high=float(row[2]), + # low=float(row[3]), + # close=float(row[4]), + # volume=float(row[5]) + # ) + # ohlcv_data.append(ohlcv) + # + # # self.save_ohlcv_data(ohlcv_data) + # current_start = chunk_end + # + # except Exception as e: + # raise RuntimeError(f"Failed to download OHLCV data: {e}") + + # Placeholder implementation - generates synthetic data using only raw Python + import random + import math + + # Calculate number of periods based on timeframe + timeframe_minutes = { + "1m": 1, "5m": 5, "15m": 15, "30m": 30, + "1h": 60, "4h": 240, "1d": 1440 + } + + minutes = timeframe_minutes.get(self.timeframe or "1h", 60) + total_minutes = int((time_to - time_from).total_seconds() / 60) + periods = total_minutes // minutes + + if periods <= 0: + return + + # Generate timestamps + delta = timedelta(minutes=minutes) + timestamps = [] + current_time = time_from + for _ in range(periods): + timestamps.append(current_time) + current_time += delta + if current_time > time_to: + break + + # Simple random walk for price data using raw Python + random.seed(42) # For reproducible data + base_price = 1.1000 # Starting price + current_price = base_price + + # Generate OHLCV data + ohlcv_data = [] + for i, ts in enumerate(timestamps): + # Simple price movement simulation + price_change = random.gauss(0, 0.001) # Normal distribution + current_price += price_change + + volatility = random.uniform(0.0005, 0.002) + open_price = current_price + close_price = current_price + random.gauss(0, volatility) + + # Ensure high >= max(open, close) and low <= min(open, close) + base_high = max(open_price, close_price) + base_low = min(open_price, close_price) + + high_price = base_high + random.uniform(0, volatility) + low_price = base_low - random.uniform(0, volatility) + volume = random.uniform(1000, 10000) + + # Create OHLCV object with Unix timestamp in milliseconds + ohlcv = OHLCV( + timestamp=int(ts.timestamp() * 1000), # Convert to milliseconds + open=round(open_price, 5), + high=round(high_price, 5), + low=round(low_price, 5), + close=round(close_price, 5), + volume=round(volume, 2) + ) + ohlcv_data.append(ohlcv) + + # Update progress callback + if on_progress and i % max(1, len(timestamps) // 10) == 0: + on_progress(ts) + + # Save data using PyneCore's save method + self.save_ohlcv_data(ohlcv_data) \ No newline at end of file diff --git a/src/pynecore/cli/templates/plugins/provider/tests/__init__.py b/src/pynecore/cli/templates/plugins/provider/tests/__init__.py new file mode 100644 index 0000000..7559d5e --- /dev/null +++ b/src/pynecore/cli/templates/plugins/provider/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for {{plugin_name_pascal}} Provider Plugin""" \ No newline at end of file diff --git a/src/pynecore/cli/templates/plugins/provider/tests/test_provider.py b/src/pynecore/cli/templates/plugins/provider/tests/test_provider.py new file mode 100644 index 0000000..75e9a3f --- /dev/null +++ b/src/pynecore/cli/templates/plugins/provider/tests/test_provider.py @@ -0,0 +1,245 @@ +"""Tests for {{plugin_name_pascal}}Provider""" + +import pytest +from datetime import datetime, timezone +from pathlib import Path +import tempfile + +from {{plugin_name_snake}}_provider import {{plugin_name_pascal}}Provider +from pynecore.types.ohlcv import OHLCV +from pynecore.core.syminfo import SymInfo + + +class Test{{plugin_name_pascal}}Provider: + """Test suite for {{plugin_name_pascal}}Provider""" + + @pytest.fixture + def temp_workdir(self): + """Create temporary working directory for tests""" + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + + @pytest.fixture + def provider(self, temp_workdir): + """Create provider instance for testing""" + # Create config directory and providers.toml + config_dir = temp_workdir / "config" + config_dir.mkdir(exist_ok=True) + + # Create a minimal providers.toml file + providers_toml = config_dir / "providers.toml" + providers_toml.write_text("[{{plugin_name_snake}}]\n# Configuration for {{plugin_name}} provider\n") + + return {{plugin_name_pascal}}Provider( + symbol="EURUSD", + timeframe="1h", + ohlv_dir=temp_workdir / "data", + config_dir=config_dir + ) + + def test_initialization(self, provider): + """Test provider initialization""" + assert provider.symbol == "EURUSD" + assert provider.timeframe == "1h" + assert provider.timezone == timezone.utc + assert isinstance(provider.config_keys, dict) + + def test_timeframe_conversion_to_tradingview(self, provider): + """Test timeframe conversion to TradingView format""" + test_cases = { + "1m": "1", + "5m": "5", + "15m": "15", + "30m": "30", + "1h": "60", + "4h": "240", + "1d": "1D", + "1w": "1W", + "1M": "1M", + } + + for provider_tf, expected_tv_tf in test_cases.items(): + result = {{plugin_name_pascal}}Provider.to_tradingview_timeframe(provider_tf) + assert result == expected_tv_tf, f"Failed for {provider_tf}: expected {expected_tv_tf}, got {result}" + + def test_timeframe_conversion_to_exchange(self, provider): + """Test timeframe conversion to exchange format""" + test_cases = { + "1": "1m", + "5": "5m", + "15": "15m", + "30": "30m", + "60": "1h", + "240": "4h", + "1D": "1d", + "1W": "1w", + "1M": "1M", + } + + for tv_tf, expected_provider_tf in test_cases.items(): + result = {{plugin_name_pascal}}Provider.to_exchange_timeframe(tv_tf) + assert result == expected_provider_tf, f"Failed for {tv_tf}: expected {expected_provider_tf}, got {result}" + + def test_get_list_of_symbols(self, provider): + """Test symbol listing""" + symbols = provider.get_list_of_symbols() + assert isinstance(symbols, list) + assert len(symbols) > 0 + assert all(isinstance(symbol, str) for symbol in symbols) + + def test_update_symbol_info(self, provider): + """Test symbol information retrieval""" + symbol_info = provider.update_symbol_info() + + assert isinstance(symbol_info, SymInfo) + assert symbol_info.symbol == "EURUSD" + assert symbol_info.description is not None + assert symbol_info.type is not None + assert symbol_info.currency is not None + assert symbol_info.exchange is not None + + def test_get_opening_hours_and_sessions(self, provider): + """Test trading hours and session information""" + opening_hours, sessions, session_ends = provider.get_opening_hours_and_sessions() + + assert isinstance(opening_hours, list) + assert isinstance(sessions, list) + assert isinstance(session_ends, list) + + def test_download_ohlcv_basic(self, provider): + """Test basic OHLCV data download""" + start_date = datetime(2023, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2023, 1, 2, tzinfo=timezone.utc) + + # Track progress calls + progress_calls = [] + def progress_callback(timestamp): + progress_calls.append(timestamp) + + # Download data (this saves to provider's OHLCV storage) + provider.download_ohlcv( + start_date=start_date, + end_date=end_date, + on_progress=progress_callback + ) + + # Verify progress was called + assert len(progress_calls) > 0 + assert all(isinstance(ts, datetime) for ts in progress_calls) + + # Read back the saved data using provider's reader + data = provider.read_ohlcv_data(start_date, end_date) + + assert isinstance(data, list) + assert len(data) > 0 + + # Check first record structure + first_record = data[0] + assert isinstance(first_record, OHLCV) + + # Check data types and structure + assert isinstance(first_record.timestamp, datetime) + assert isinstance(first_record.open, (int, float)) + assert isinstance(first_record.high, (int, float)) + assert isinstance(first_record.low, (int, float)) + assert isinstance(first_record.close, (int, float)) + assert isinstance(first_record.volume, (int, float)) + + # Check OHLC relationships + for record in data: + assert record.high >= record.open + assert record.high >= record.close + assert record.high >= record.low + assert record.low <= record.open + assert record.low <= record.close + assert record.volume >= 0 + + def test_download_ohlcv_with_progress(self, provider): + """Test OHLCV download with progress callback""" + start_date = datetime(2023, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2023, 1, 2, tzinfo=timezone.utc) + + progress_calls = [] + + def progress_callback(timestamp): + progress_calls.append(timestamp) + assert isinstance(timestamp, datetime) + + provider.download_ohlcv( + start_date=start_date, + end_date=end_date, + on_progress=progress_callback + ) + + assert len(progress_calls) > 0 + # Verify all progress calls are datetime objects + assert all(isinstance(ts, datetime) for ts in progress_calls) + + # Read back the data to verify it was saved + data = provider.read_ohlcv_data(start_date, end_date) + assert isinstance(data, list) + assert len(data) > 0 + + def test_download_ohlcv_empty_range(self, provider): + """Test OHLCV download with invalid time range""" + start_date = datetime(2024, 1, 2, tzinfo=timezone.utc) + end_date = datetime(2024, 1, 1, tzinfo=timezone.utc) # Invalid range + + provider.download_ohlcv( + start_date=start_date, + end_date=end_date + ) + + # Read back the data - should be empty + data = provider.read_ohlcv_data(start_date, end_date) + assert isinstance(data, list) + assert len(data) == 0 + + def test_download_ohlcv_date_range(self, provider): + """Test OHLCV download with specific date range""" + start_date = datetime(2023, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2023, 1, 3, tzinfo=timezone.utc) + + provider.download_ohlcv( + start_date=start_date, + end_date=end_date + ) + + # Read back the saved data + data = provider.read_ohlcv_data(start_date, end_date) + + assert isinstance(data, list) + assert len(data) > 0 + + # Check that all timestamps are within the requested range + for record in data: + timestamp = record.timestamp + assert start_date <= timestamp <= end_date + + @pytest.mark.parametrize("timeframe", ["1m", "5m", "15m", "30m", "1h", "4h", "1d"]) + def test_download_ohlcv_different_timeframes(self, provider, timeframe): + """Test OHLCV download with different timeframes""" + start_date = datetime(2024, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2024, 1, 1, 6, tzinfo=timezone.utc) # 6 hours + + # Create provider with specific timeframe + provider_tf = {{plugin_name_pascal}}Provider( + symbol="EURUSD", + timeframe=timeframe, + ohlv_dir=provider.ohlv_dir, + config_dir=provider.config_dir + ) + + provider_tf.download_ohlcv( + start_date=start_date, + end_date=end_date + ) + + # Read back the data + data = provider_tf.read_ohlcv_data(start_date, end_date) + + assert isinstance(data, list) + # The exact number of records depends on the timeframe and implementation + # but we can check that the structure is correct + if data: + assert isinstance(data[0], OHLCV) \ No newline at end of file diff --git a/src/pynecore/providers/__init__.py b/src/pynecore/providers/__init__.py index db0626c..153130d 100644 --- a/src/pynecore/providers/__init__.py +++ b/src/pynecore/providers/__init__.py @@ -1,10 +1,85 @@ from pathlib import Path +from typing import Dict, Type, Optional from .ccxt import CCXTProvider from .capitalcom import CapitalComProvider +from .provider import Provider -# List of available providers -available_providers = tuple( - p.stem for p in Path(__file__).parent.resolve().glob('*.py') if - p.name not in ('__init__.py', 'provider.py') -) +# Built-in providers +_builtin_providers = { + 'ccxt': CCXTProvider, + 'capitalcom': CapitalComProvider, +} + +# Cache for loaded plugin providers +_plugin_providers: Dict[str, Type[Provider]] = {} + + +def get_available_providers() -> tuple[str, ...]: + """Get list of all available providers (built-in + plugins) + + Returns: + Tuple of provider names + """ + # Get built-in providers + builtin = set(_builtin_providers.keys()) + + # Get plugin providers + plugin_names = set() + try: + # Import here to avoid circular imports + from ..cli.plugin_manager import plugin_manager + plugin_names = set(plugin_manager.get_available_providers()) + except ImportError: + # Plugin manager not available (e.g., in minimal installations) + pass + + return tuple(sorted(builtin | plugin_names)) + + +def get_provider_class(provider_name: str) -> Optional[Type[Provider]]: + """Get provider class by name + + Args: + provider_name: Name of the provider + + Returns: + Provider class or None if not found + """ + # Check built-in providers first + if provider_name in _builtin_providers: + return _builtin_providers[provider_name] + + # Check cached plugin providers + if provider_name in _plugin_providers: + return _plugin_providers[provider_name] + + # Try to load from plugins + try: + from ..cli.plugin_manager import plugin_manager + provider_class = plugin_manager.load_plugin(provider_name) + if provider_class: + # Cache the loaded provider + _plugin_providers[provider_name] = provider_class + return provider_class + except ImportError: + # Plugin manager not available + pass + + return None + + +# Legacy support - dynamically generate available_providers +# This maintains backward compatibility with existing code +available_providers = get_available_providers() + + +# Re-export for convenience +__all__ = [ + 'Provider', + 'CCXTProvider', + 'CapitalComProvider', + 'available_providers', + 'get_available_providers', + 'get_provider_class', +] From beb3de63a233e17bdb82abd12713f208694f8cd1 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Sun, 27 Jul 2025 18:13:46 +0600 Subject: [PATCH 2/7] chore: ignore IDE-specific files in .gitignore Add .vscode/ and .idea/ directories to .gitignore to exclude IDE-specific files from version control --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 4a6e11c..ba7f9c3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,7 @@ dist/ # macOS # .DS_Store + +# ide files +.vscode/ +.idea/ \ No newline at end of file From 4b3c23d96165b5f591374d1550c2d725dd096afd Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Mon, 28 Jul 2025 15:47:52 +0600 Subject: [PATCH 3/7] fix(template_engine): improve version fetching with PyPI fallback Modify version fetching logic to first attempt getting version from PyPI API before falling back to local pyproject.toml. This provides more reliable version detection in distributed environments. Also fix regex pattern string in template engine by making it a raw string. --- src/pynecore/cli/template_engine.py | 36 +++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/pynecore/cli/template_engine.py b/src/pynecore/cli/template_engine.py index 34f8019..2757181 100644 --- a/src/pynecore/cli/template_engine.py +++ b/src/pynecore/cli/template_engine.py @@ -20,7 +20,7 @@ def render(self, template_content: str) -> str: # Replace variables in format {{variable_name}} for key, value in self.variables.items(): - pattern = f"{{{{\s*{key}\s*}}}}" + pattern = rf"{{{{\s*{key}\s*}}}}" result = re.sub(pattern, str(value), result) return result @@ -67,11 +67,30 @@ def get_default_template_variables(plugin_name: str, plugin_type: str) -> Dict[s def _get_pynecore_version() -> str: - """Get PyneCore version from main pyproject.toml""" + """Get PyneCore version from PyPI""" + try: + import subprocess + import json + + # Use curl to fetch version from PyPI API (avoids SSL issues) + url = "https://pypi.org/pypi/pynesys-pynecore/json" + result = subprocess.run( + ["curl", "-s", "--max-time", "10", url], + capture_output=True, + text=True, + timeout=15 + ) + + if result.returncode == 0 and result.stdout: + data = json.loads(result.stdout) + version = data.get('info', {}).get('version', '0.1.0') + return f">={version}" + except Exception: + pass + + # Fallback on any error - try to get from local pyproject.toml try: - # Get the path to the main pyproject.toml (relative to this file) current_file = Path(__file__) - # Navigate up to the project root: cli/template_engine.py -> cli -> pynecore -> src -> root project_root = current_file.parent.parent.parent.parent pyproject_path = project_root / "pyproject.toml" @@ -81,9 +100,8 @@ def _get_pynecore_version() -> str: data = tomllib.load(f) version = data.get('project', {}).get('version', '0.1.0') return f">={version}" - else: - # Fallback if pyproject.toml not found - return ">=0.1.0" except Exception: - # Fallback on any error - return ">=0.1.0" \ No newline at end of file + pass + + # Final fallback + return ">=0.1.0" \ No newline at end of file From 61c8e92d157b9f917eb8d8232edb9fa423df4d2d Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 21:57:39 +0600 Subject: [PATCH 4/7] fix(cli): correct typo in plugins command name from plugins to plugin --- src/pynecore/cli/commands/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynecore/cli/commands/plugins.py b/src/pynecore/cli/commands/plugins.py index 2c33ae5..51c383c 100644 --- a/src/pynecore/cli/commands/plugins.py +++ b/src/pynecore/cli/commands/plugins.py @@ -16,7 +16,7 @@ from ..plugin_manager import plugin_manager app_plugins = Typer(help="Plugin management commands") -app.add_typer(app_plugins, name="plugins") +app.add_typer(app_plugins, name="plugin") console = Console() From b945843d127db8226670d3490b1d2d37c090f808 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 22:06:44 +0600 Subject: [PATCH 5/7] fix(cli): skip workdir setup for plugin commands Plugin commands don't require workdir setup, so skip it to avoid unnecessary operations --- src/pynecore/cli/commands/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pynecore/cli/commands/__init__.py b/src/pynecore/cli/commands/__init__.py index 917f2fc..64b3295 100644 --- a/src/pynecore/cli/commands/__init__.py +++ b/src/pynecore/cli/commands/__init__.py @@ -38,6 +38,10 @@ def setup( return if any(arg in ('-h', '--help') for arg in sys.argv[1:]): return + + # Skip workdir setup for plugin commands as they don't need it + if ctx.invoked_subcommand == 'plugin': + return typer.echo("") From b1d5889be33bfce195fc7530ec9472b5b002aaf8 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 22:10:50 +0600 Subject: [PATCH 6/7] chore: remove ide-specific files from gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index ba7f9c3..4a6e11c 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,3 @@ dist/ # macOS # .DS_Store - -# ide files -.vscode/ -.idea/ \ No newline at end of file From 6854e69aa1b68e2aaacb1afee1a2381a7df52f08 Mon Sep 17 00:00:00 2001 From: Mahadi Hassan Date: Wed, 30 Jul 2025 22:24:15 +0600 Subject: [PATCH 7/7] refactor(plugin): move plugin system to core module for better organization Centralize plugin management functionality in the core module to improve code structure and avoid circular imports. This change makes the plugin system available to all PyneCore applications, not just the CLI. --- src/pynecore/cli/commands/plugins.py | 2 +- src/pynecore/core/plugin/__init__.py | 9 +++++++++ src/pynecore/{cli => core/plugin}/plugin_manager.py | 2 +- src/pynecore/providers/__init__.py | 4 ++-- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 src/pynecore/core/plugin/__init__.py rename src/pynecore/{cli => core/plugin}/plugin_manager.py (99%) diff --git a/src/pynecore/cli/commands/plugins.py b/src/pynecore/cli/commands/plugins.py index 51c383c..8710ae3 100644 --- a/src/pynecore/cli/commands/plugins.py +++ b/src/pynecore/cli/commands/plugins.py @@ -13,7 +13,7 @@ from ..app import app, app_state from ..template_engine import TemplateEngine, get_plugin_templates_dir, get_default_template_variables -from ..plugin_manager import plugin_manager +from ...core.plugin import plugin_manager app_plugins = Typer(help="Plugin management commands") app.add_typer(app_plugins, name="plugin") diff --git a/src/pynecore/core/plugin/__init__.py b/src/pynecore/core/plugin/__init__.py new file mode 100644 index 0000000..5c739ac --- /dev/null +++ b/src/pynecore/core/plugin/__init__.py @@ -0,0 +1,9 @@ +"""Plugin system for PyneCore + +This module provides plugin discovery, registration, and management capabilities +that can be used by any PyneCore application, not just the CLI. +""" + +from .plugin_manager import PluginManager, PluginInfo, plugin_manager + +__all__ = ['PluginManager', 'PluginInfo', 'plugin_manager'] \ No newline at end of file diff --git a/src/pynecore/cli/plugin_manager.py b/src/pynecore/core/plugin/plugin_manager.py similarity index 99% rename from src/pynecore/cli/plugin_manager.py rename to src/pynecore/core/plugin/plugin_manager.py index ea698c9..4a0a893 100644 --- a/src/pynecore/cli/plugin_manager.py +++ b/src/pynecore/core/plugin/plugin_manager.py @@ -120,7 +120,7 @@ def _discover_builtin_providers(self) -> Dict[str, PluginInfo]: try: # Import built-in providers info - from ..providers import _builtin_providers + from ...providers import _builtin_providers for name, provider_class in _builtin_providers.items(): # Get version from the main package diff --git a/src/pynecore/providers/__init__.py b/src/pynecore/providers/__init__.py index 153130d..70a1461 100644 --- a/src/pynecore/providers/__init__.py +++ b/src/pynecore/providers/__init__.py @@ -28,7 +28,7 @@ def get_available_providers() -> tuple[str, ...]: plugin_names = set() try: # Import here to avoid circular imports - from ..cli.plugin_manager import plugin_manager + from ..core.plugin import plugin_manager plugin_names = set(plugin_manager.get_available_providers()) except ImportError: # Plugin manager not available (e.g., in minimal installations) @@ -56,7 +56,7 @@ def get_provider_class(provider_name: str) -> Optional[Type[Provider]]: # Try to load from plugins try: - from ..cli.plugin_manager import plugin_manager + from ..core.plugin import plugin_manager provider_class = plugin_manager.load_plugin(provider_name) if provider_class: # Cache the loaded provider