Merge pull request #18 from linkml/enhancements

enhancements

Author: cmungall

Makefile

@@ -21,13 +21,20 @@ all-pytest:
 install-all:
 	poetry install -E analytics -E app -E tests -E llm -E mongodb
 
-# not yet deployed
+DOCTEST_DIR = docs src/linkml_store/api src/linkml_store/index src/linkml_store/utils
 doctest:
-	find src docs -type f \( -name "*.rst" -o -name "*.md" -o -name "*.py" \) -print0 | xargs -0 $(RUN) python -m doctest --option ELLIPSIS --option NORMALIZE_WHITESPACE
+	find $(DOCTEST_DIR) -type f \( -name "*.rst" -o -name "*.md" -o -name "*.py" \) -print0 | xargs -0 $(RUN) python -m doctest --option ELLIPSIS --option NORMALIZE_WHITESPACE
+
+
+doctest-%:
+	find $* -type f \( -name "*.py" \) -print0 | xargs -0 $(RUN) python -m doctest --option ELLIPSIS --option NORMALIZE_WHITESPACE
 
 %-doctest: %
 	$(RUN) python -m doctest --option ELLIPSIS --option NORMALIZE_WHITESPACE $<
 
+api:
+	poetry run linkml-store-api
+
 app:
 	$(RUN) streamlit run $(CODE)/app.py
 #	$(RUN) streamlit run $(CODE)/app.py --logger.level=debug

README.md

@@ -1,12 +1,84 @@
 # linkml-store
 
-An integration layer for multiple database backends
+An AI-ready data management and integration platform. LinkML-Store
+provides an abstraction layer over multiple different backends
+(including DuckDB, MongoDB, and local filesystems), allowing for
+common query, index, and storage operations.
 
-Currently this software is alpha, and has only been tested with DuckDB backends. See the Tutorial for more information.
+For full documentation, see [https://linkml.io/linkml-store/](https://linkml.io/linkml-store/)
 
-There is also experimental support for vector-based indexing using OpenAI test embedding via the `llm` library.
+__Warning__ LinkML-Store is still undergoing changes and refactoring,
+APIs and command line options are subject to change!
 
-The goals of this project are to provide high level access to data stored in heterogeneous databases,
-with optional schema management using LinkML.
+## Quick Start
+
+Install, add data, query it:
+
+```
+pip install linkml-store[all]
+linkml-store -d duckdb:///db/my.db -c persons insert data/*.json
+linkml-store -d duckdb:///db/my.db -c persons query -w "occupation: Bricklayer"
+```
+
+Index it, search it:
+
+```
+linkml-store -d duckdb:///db/my.db -c persons index -t llm
+linkml-store -d duckdb:///db/my.db -c persons search "all persons employed in construction"
+```
+
+Validate it:
+
+```
+linkml-store -d duckdb:///db/my.db -c persons validate
+```
+
+## Basic usage
+
+* [Command Line](https://linkml.io/linkml-store/tutorials/Command-Line-Tutorial.html)
+* [Python](https://linkml.io/linkml-store/tutorials/Python-Tutorial.html)
+* API
+* Streamlit applications
+
+## Features
+
+### Multiple Adapters
+
+LinkML-Store is designed to work with multiple backends, giving a common abstraction layer
+
+* [MongoDB](https://linkml.io/linkml-store/how-to/Use-MongoDB.html)
+* [DuckDB](https://linkml.io/linkml-store/tutorials/Python-Tutorial.html)
+* [Solr](https://linkml.io/linkml-store/how-to/Query-Solr-using-CLI.html)
+* Filesystem
+
+Coming soon: any RDBMS, any triplestore, Neo4J, HDF5-based stores, ChromaDB/Vector dbs ...
+
+The intent is to give a union of all features of each backend. For
+example, analytic faceted queries are provided for *all* backends, not
+just Solr.
+
+### Composable indexes
+
+Many backends come with their own indexing and search
+schemes. Classically this was Lucene-based indexes, now it is semantic
+search using LLM embeddings.
+
+LinkML store treats indexing as an orthogonal concern - you can
+compose different indexing schemes with different backends. You don't
+need to have a vector database to run embedding search!
+
+See [How to Use-Semantic-Search](https://linkml.io/linkml-store/how-to/Use-Semantic-Search.html)
+
+### Validation
+
+LinkML-Store is backed by [LinkML](https://linkml.io), which allows
+for powerful expressive structural and semantic constraints.
+
+See [Indexing JSON](https://linkml.io/linkml-store/how-to/Index-Phenopackets.html)
+
+and [Referential Integrity](https://linkml.io/linkml-store/how-to/Check-Referential-Integrity.html)
+
+
+## Background
 
 See [these slides](https://docs.google.com/presentation/d/e/2PACX-1vSgtWUNUW0qNO_ZhMAGQ6fYhlXZJjBNMYT0OiZz8DDx8oj7iG9KofRs6SeaMXBBOICGknoyMG2zaHnm/embed?start=false&loop=false&delayms=3000) for more details

docs/about.rst

@@ -17,10 +17,26 @@ The overall goals are to provide:
 Installation
 ------------
 
+At this stage we recommend installing all extras:
+
+.. code-block:: bash
+
+    pip install "linkml-store[all]"
+
+Minimal implementation:
+
 .. code-block:: bash
 
-    pip install linkml-store
+    pip install "linkml-store"
+
+For developers working on the linkml-store codebase, we recommend checking out the repo, and then
+installing all packages via the `make` command:
+
+.. code-block:: bash
 
+    git clone <URL>
+    cd linkml-store
+    make install
 
 Quickstart
 ----------