Skip to content

Commit 2cf0445

Browse files
arunp77arunp77
authored
Update data_etl_tools.html
1 parent 3e82025 commit 2cf0445

File tree

1 file changed

+190
-0
lines changed

1 file changed

+190
-0
lines changed

‎data_etl_tools.html

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,191 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
13

4+
<head>
5+
<meta charset="utf-8">
6+
<meta content="width=device-width, initial-scale=1.0" name="viewport">
7+
8+
<title>Satellite data monitoring tools</title>
9+
<meta content="" name="description">
10+
<meta content="" name="keywords">
11+
12+
<!-- Favicons -->
13+
<link href="assets/img/Favicon-1.png" rel="icon">
14+
<link href="assets/img/Favicon-1.png" rel="apple-touch-icon">
15+
16+
<!-- Google Fonts -->
17+
<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,300i,400,400i,600,600i,700,700i|Raleway:300,300i,400,400i,500,500i,600,600i,700,700i|Poppins:300,300i,400,400i,500,500i,600,600i,700,700i" rel="stylesheet">
18+
19+
<!-- Vendor CSS Files -->
20+
<link href="assets/vendor/aos/aos.css" rel="stylesheet">
21+
<link href="assets/vendor/bootstrap/css/bootstrap.min.css" rel="stylesheet">
22+
<link href="assets/vendor/bootstrap-icons/bootstrap-icons.css" rel="stylesheet">
23+
<link href="assets/vendor/boxicons/css/boxicons.min.css" rel="stylesheet">
24+
<link href="assets/vendor/glightbox/css/glightbox.min.css" rel="stylesheet">
25+
<link href="assets/vendor/swiper/swiper-bundle.min.css" rel="stylesheet">
26+
<!-- Creating a python code section-->
27+
<link rel="stylesheet" href="assets/css/prism.css">
28+
<script src="assets/js/prism.js"></script>
29+
30+
<!-- Template Main CSS File -->
31+
<link href="assets/css/style.css" rel="stylesheet">
32+
33+
<!-- To set the icon, visit https://fontawesome.com/account-->
34+
<script src="https://kit.fontawesome.com/5d25c1efd3.js" crossorigin="anonymous"></script>
35+
<!-- end of icon-->
36+
37+
<!-- =======================================================
38+
* Template Name: iPortfolio
39+
* Updated: Sep 18 2023 with Bootstrap v5.3.2
40+
* Template URL: https://bootstrapmade.com/iportfolio-bootstrap-portfolio-websites-template/
41+
* Author: BootstrapMade.com
42+
* License: https://bootstrapmade.com/license/
43+
======================================================== -->
44+
</head>
45+
46+
<body>
47+
48+
<!-- ======= Mobile nav toggle button ======= -->
49+
<i class="bi bi-list mobile-nav-toggle d-xl-none"></i>
50+
51+
<!-- ======= Header ======= -->
52+
<header id="header">
53+
<div class="d-flex flex-column">
54+
55+
<div class="profile">
56+
<img src="assets/img/myphoto.jpeg" alt="" class="img-fluid rounded-circle">
57+
<h1 class="text-light"><a href="index.html">Arun</a></h1>
58+
<div class="social-links mt-3 text-center">
59+
<a href="https://www.linkedin.com/in/arunp77/" target="_blank" class="linkedin"><i class="bx bxl-linkedin"></i></a>
60+
<a href="https://github.com/arunp77" target="_blank" class="github"><i class="bx bxl-github"></i></a>
61+
<a href="https://twitter.com/arunp77_" target="_blank" class="twitter"><i class="bx bxl-twitter"></i></a>
62+
<a href="https://www.instagram.com/arunp77/" target="_blank" class="instagram"><i class="bx bxl-instagram"></i></a>
63+
<a href="https://arunp77.medium.com/" target="_blank" class="medium"><i class="bx bxl-medium"></i></a>
64+
</div>
65+
</div>
66+
67+
<nav id="navbar" class="nav-menu navbar">
68+
<ul>
69+
<li><a href="index.html#hero" class="nav-link scrollto active"><i class="bx bx-home"></i> <span>Home</span></a></li>
70+
<li><a href="index.html#about" class="nav-link scrollto"><i class="bx bx-user"></i> <span>About</span></a></li>
71+
<li><a href="index.html#resume" class="nav-link scrollto"><i class="bx bx-file-blank"></i> <span>Resume</span></a></li>
72+
<li><a href="index.html#portfolio" class="nav-link scrollto"><i class="bx bx-book-content"></i> <span>Portfolio</span></a></li>
73+
<li><a href="index.html#skills-and-tools" class="nav-link scrollto"><i class="bx bx-wrench"></i> <span>Skills and Tools</span></a></li>
74+
<li><a href="index.html#language" class="nav-link scrollto"><i class="bi bi-menu-up"></i> <span>Languages</span></a></li>
75+
<li><a href="index.html#awards" class="nav-link scrollto"><i class="bi bi-award-fill"></i> <span>Awards</span></a></li>
76+
<li><a href="index.html#professionalcourses" class="nav-link scrollto"><i class="bx bx-book-alt"></i> <span>Professional Certification</span></a></li>
77+
<li><a href="index.html#publications" class="nav-link scrollto"><i class="bx bx-news"></i> <span>Publications</span></a></li>
78+
<li><a href="index.html#extra-curricular" class="nav-link scrollto"><i class="bx bx-rocket"></i> <span>Extra-Curricular Activities</span></a></li>
79+
<!-- <li><a href="#contact" class="nav-link scrollto"><i class="bx bx-envelope"></i> <span>Contact</span></a></li> -->
80+
</ul>
81+
</nav><!-- .nav-menu -->
82+
</div>
83+
</header><!-- End Header -->
84+
85+
<main id="main">
86+
87+
<!-- ======= Breadcrumbs ======= -->
88+
<section id="breadcrumbs" class="breadcrumbs">
89+
<div class="container">
90+
91+
<div class="d-flex justify-content-between align-items-center">
92+
<h2>Portfolio Details</h2>
93+
<ol>
94+
<li><a href="portfolio-details-8.html" class="clickable-box">Go to Project page <i class="fas fa-arrow-right"></i></a></li>
95+
<li><a href="index.html#portfolio" class="clickable-box">Go to portfolio</a></li>
96+
</ol>
97+
</div>
98+
99+
</div>
100+
</section><!-- End Breadcrumbs -->
101+
102+
<!------ right dropdown menue ------->
103+
<div class="right-side-list">
104+
<div class="dropdown">
105+
<button class="dropbtn"><strong>Shortcuts:</strong></button>
106+
<div class="dropdown-content">
107+
<ul>
108+
<li><a href="cloud-compute.html"><i class="fas fa-cloud"></i> Cloud</a></li>
109+
<li><a href="AWS-GCP.html"><i class="fas fa-cloud"></i> AWS-GCP</a></li>
110+
<li><a href="amazon-s3.html"><i class="fas fa-cloud"></i> AWS S3</a></li>
111+
<li><a href="ec2-confi.html"><i class="fas fa-server"></i> EC2</a></li>
112+
<li><a href="Docker-Container.html"><i class="fab fa-docker" style="color: rgb(8, 8, 8);"></i> Docker</a></li>
113+
<li><a href="Jupyter-nifi.html"><i class="fab fa-python" style="color: rgb(8, 8, 8);"></i> Jupyter-nifi</a></li>
114+
<li><a href="snowflake-task-stream.html"><i class="fas fa-snowflake"></i> Snowflake</a></li>
115+
<li><a href="data-model.html"><i class="fas fa-database"></i> Data modeling</a></li>
116+
<li><a href="sql-basics.html"><i class="fas fa-table"></i> QL</a></li>
117+
<li><a href="sql-basic-details.html"><i class="fas fa-database"></i> SQL</a></li>
118+
<li><a href="Bigquerry-sql.html"><i class="fas fa-database"></i> Bigquerry</a></li>
119+
<li><a href="scd.html"><i class="fas fa-archive"></i> SCD</a></li>
120+
<li><a href="sql-project.html"><i class="fas fa-database"></i> SQL project</a></li>
121+
<!-- Add more subsections as needed -->
122+
</ul>
123+
</div>
124+
</div>
125+
</div>
126+
127+
<!-- ======= Portfolio Details Section ======= -->
128+
<section id="portfolio-details" class="portfolio-details">
129+
<div class="container">
130+
<div class="row gy-4">
131+
<h1>Satellite data processing and anomaly monitoring tools</h1>
132+
<div class="image">
133+
<img src="assets/img/portfolio/docker-logo.png" alt="Image Description" width="300" height="200">
134+
</div>
135+
136+
<h3>🧊Full-stack Flask application</h3>
137+
<ul>
138+
<li>Developed a full-stack Flask application and deployed on a linux server for real-time product monitoring, incorporating advanced file processing to extract and analyze remote sensing products, including L0, L1, and L2. </li>
139+
<li>The tool facilitates the comparison of product availability between the EUMETSAT Data Lake and the data received from ground stations, ensuring seamless integration and synchronization of satellite data for accurate monitoring and analysis.</li>
140+
<li>The tool can also monitor each L0, L1 and L2 product file size and compare them with the expected size with respect to requirement documents. </li>
141+
<li>Designed and implemented a NoSQL Elasticsearch database to efficiently store and manage structured log data for advanced querying and analysis.</li>
142+
<li>Automated data ingestion with scheduled tasks to ensure continuous updates, keeping the database synchronized with the latest logs.</li>
143+
<li>Built dynamic dashboards with interactive and static visualizations, leveraging processed log data to enhance system monitoring and decision-making.</li>
144+
<li>Created and configured customized Kibana dashboards using high-level Vega scripting, integrated with Elasticsearch to monitor and visualize size of L0, L1, and L2 products. The dashboards include interactive filters, time-series visualizations, and alerting features to detect size deviations and support timely troubleshooting.</li>
145+
<li>Integrated end-to-end workflows, from raw data extraction and transformation to database storage and real-time analytics, optimizing operational efficiency.</li>
146+
</ul>
147+
148+
<h3>🧊 Telemetry packet analysis tool</h3>
149+
<ul>
150+
<li>Developed a Jupyter Notebook-based tool to analyze telemetry packets from L0 satellite products, focusing on the detection of missing or anomalous packets. The tool aids in identifying unusual patterns and behaviors in specific products. Integrated the solution with the tm-packet-analyser utility to extract detailed packet-level information for in-depth analysis.🧊Full-stack Flask application: ✔ Developed a full-stack Flask application and deployed on a linux server for real-time product monitoring, incorporating advanced file processing to extract and analyze remote sensing products, including L0, L1, and L2. ✔ The tool facilitates the comparison of product availability between the EUMETSAT Data Lake and the data received from ground stations, ensuring seamless integration and synchronization of satellite data for accurate monitoring and analysis. ✔ The tool can also monitor each L0, L1 and L2 product file size and compare them with the expected size with respect to requirement documents. ✔ Designed and implemented a NoSQL Elasticsearch database to efficiently store and manage structured log data for advanced querying and analysis. ✔ Automated data ingestion with scheduled tasks to ensure continuous updates, keeping the database synchronized with the latest logs. ✔ Built dynamic dashboards with interactive and static visualizations, leveraging processed log data to enhance system monitoring and decision-making. ✔ Created and configured customized Kibana dashboards using high-level Vega scripting, integrated with Elasticsearch to monitor and visualize size of L0, L1, and L2 products. The dashboards include interactive filters, time-series visualizations, and alerting features to detect size deviations and support timely troubleshooting. ✔ Integrated end-to-end workflows, from raw data extraction and transformation to database storage and real-time analytics, optimizing operational efficiency. 🧊 Telemetry packet analysis tool: ✔ Developed a Jupyter Notebook-based tool to analyze telemetry packets from L0 satellite products, focusing on the detection of missing or anomalous packets. The tool aids in identifying unusual patterns and behaviors in specific products. Integrated the solution with the tm-packet-analyser utility to extract detailed packet-level information for in-depth analysis.
151+
Bash, Linux and +16 skills</li>
152+
</ul>
153+
</div>
154+
</div>
155+
</section><!-- End Portfolio Details Section -->
156+
</main><!-- End #main -->
157+
158+
<!-- ======= Footer ======= -->
159+
<footer id="footer">
160+
<div class="container">
161+
<div class="copyright">
162+
&copy; Copyright <strong><span>Arun</span></strong>
163+
</div>
164+
</div>
165+
</footer><!-- End Footer -->
166+
167+
<a href="#" class="back-to-top d-flex align-items-center justify-content-center"><i class="bi bi-arrow-up-short"></i></a>
168+
169+
<!-- Vendor JS Files -->
170+
<script src="assets/vendor/purecounter/purecounter_vanilla.js"></script>
171+
<script src="assets/vendor/aos/aos.js"></script>
172+
<script src="assets/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
173+
<script src="assets/vendor/glightbox/js/glightbox.min.js"></script>
174+
<script src="assets/vendor/isotope-layout/isotope.pkgd.min.js"></script>
175+
<script src="assets/vendor/swiper/swiper-bundle.min.js"></script>
176+
<script src="assets/vendor/typed.js/typed.umd.js"></script>
177+
<script src="assets/vendor/waypoints/noframework.waypoints.js"></script>
178+
<script src="assets/vendor/php-email-form/validate.js"></script>
179+
180+
<!-- Template Main JS File -->
181+
<script src="assets/js/main.js"></script>
182+
183+
<script>
184+
document.addEventListener("DOMContentLoaded", function () {
185+
hljs.initHighlightingOnLoad();
186+
});
187+
</script>
188+
189+
</body>
190+
191+
</html>

0 commit comments

Comments
 (0)