"use strict";(self.webpackChunkdocs_openc3_com=self.webpackChunkdocs_openc3_com||[]).push([[5423],{6506:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>i,default:()=>h,frontMatter:()=>r,metadata:()=>a,toc:()=>l});var o=t(2322),s=t(5392);const r={title:"Monitoring"},i=void 0,a={id:"guides/monitoring",title:"Monitoring",description:"Monitoring and observability",source:"@site/docs/guides/monitoring.md",sourceDirName:"guides",slug:"/guides/monitoring",permalink:"/tools/staticdocs/docs/guides/monitoring",draft:!1,unlisted:!1,editUrl:"https://github.com/OpenC3/cosmos/tree/main/docs.openc3.com/docs/guides/monitoring.md",tags:[],version:"current",frontMatter:{title:"Monitoring"},sidebar:"defaultSidebar",previous:{title:"Logging",permalink:"/tools/staticdocs/docs/guides/logging"},next:{title:"Performance",permalink:"/tools/staticdocs/docs/guides/performance"}},c={},l=[{value:"Monitoring and observability",id:"monitoring-and-observability",level:3},{value:"Fluent/Fluentd",id:"fluentfluentd",level:3},{value:"Notes",id:"notes",level:4},{value:"OpenDistro",id:"opendistro",level:3},{value:"Notes",id:"notes-1",level:4},{value:"Prometheus",id:"prometheus",level:3},{value:"Notes",id:"notes-2",level:4},{value:"Grafana",id:"grafana",level:3},{value:"Notes",id:"notes-3",level:4}];function d(e){const n={a:"a",blockquote:"blockquote",code:"code",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",ul:"ul",...(0,s.a)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)(n.h3,{id:"monitoring-and-observability",children:"Monitoring and observability"}),"\n",(0,o.jsxs)(n.p,{children:["With moving COSMOS to container based service, we needed a better way to monitor the internals of COSMOS. So here is some information on external services that you can use to monitor COSMOS. If you want to read more about ",(0,o.jsx)(n.a,{href:"https://sre.google/sre-book/monitoring-distributed-systems/",children:"Monitoring Distributed Systems"})]}),"\n",(0,o.jsx)(n.h3,{id:"fluentfluentd",children:(0,o.jsx)(n.a,{href:"https://www.fluentd.org/guides/recipes/docker-logging",children:"Fluent/Fluentd"})}),"\n",(0,o.jsxs)(n.blockquote,{children:["\n",(0,o.jsx)(n.p,{children:"Fluentd is an open source data collector, which lets you unify the data collection and consumption for a better use and understanding of data."}),"\n"]}),"\n",(0,o.jsx)(n.h4,{id:"notes",children:"Notes"}),"\n",(0,o.jsx)(n.p,{children:"in_docker.conf"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"<source>\n  @type forward\n  port 24224\n  bind 0.0.0.0\n</source>\n<match *.metric>\n  @type copy\n  <store>\n    @type elasticsearch\n    host openc3-elasticsearch\n    port 9200\n    logstash_format true\n    logstash_prefix metric\n    logstash_dateformat %Y%m%d\n    include_tag_key true\n    type_name access_log\n    tag_key @log_name\n    flush_interval 1s\n  </store>\n  <store>\n    @type stdout\n  </store>\n</match>\n<match *__openc3.log>\n  @type copy\n  <store>\n    @type elasticsearch\n    host openc3-elasticsearch\n    port 9200\n    logstash_format true\n    logstash_prefix openc3\n    logstash_dateformat %Y%m%d\n    include_tag_key true\n    type_name access_log\n    tag_key @log_name\n    flush_interval 1s\n  </store>\n  <store>\n    @type stdout\n  </store>\n</match>\n<match *.**>\n  @type copy\n  <store>\n    @type elasticsearch\n    host openc3-elasticsearch\n    port 9200\n    logstash_format true\n    logstash_prefix fluentd\n    logstash_dateformat %Y%m%d\n    include_tag_key true\n    type_name access_log\n    tag_key @log_name\n    flush_interval 1s\n  </store>\n  <store>\n    @type stdout\n  </store>\n</match>\n"})}),"\n",(0,o.jsx)(n.p,{children:"Dockerfile"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"FROM fluent/fluentd:v1.10.3-1.0\n\nCOPY ./in_docker.conf /fluentd/etc/fluent.conf\nUSER root\nRUN gem install fluent-plugin-elasticsearch --no-document --version 4.0.7 \\\n  && gem install fluent-plugin-prometheus --no-document --version 1.8.5\nUSER fluent\n"})}),"\n",(0,o.jsx)(n.h3,{id:"opendistro",children:(0,o.jsx)(n.a,{href:"https://opendistro.github.io/for-elasticsearch-docs/",children:"OpenDistro"})}),"\n",(0,o.jsxs)(n.blockquote,{children:["\n",(0,o.jsx)(n.p,{children:"Open Distro for Elasticsearch provides a powerful, easy-to-use event monitoring and alerting system, enabling you to monitor your data and send notifications automatically to your stakeholders. With an intuitive Kibana interface and powerful API, it is easy to set up and manage alerts."}),"\n"]}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsx)(n.li,{children:(0,o.jsx)(n.a,{href:"https://opendistro.github.io/for-elasticsearch-docs/docs/install/docker/",children:"Docker"})}),"\n"]}),"\n",(0,o.jsx)(n.h4,{id:"notes-1",children:"Notes"}),"\n",(0,o.jsx)(n.p,{children:"When testing this I found that depending on how you ingest your logs into the opendistro I found I had to disable security. Here is an example of the docker file."}),"\n",(0,o.jsx)(n.p,{children:"Dockerfile"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"FROM amazon/opendistro-for-elasticsearch:1.12.0\n\nRUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security\n"})}),"\n",(0,o.jsx)(n.h3,{id:"prometheus",children:(0,o.jsx)(n.a,{href:"https://prometheus.io/",children:"Prometheus"})}),"\n",(0,o.jsxs)(n.blockquote,{children:["\n",(0,o.jsx)(n.p,{children:"Prometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. It stores all scraped samples locally and runs rules over this data to either aggregate and record new time series from existing data or generate alerts. Grafana or other API consumers can be used to visualize the collected data."}),"\n"]}),"\n",(0,o.jsx)(n.h4,{id:"notes-2",children:"Notes"}),"\n",(0,o.jsx)(n.p,{children:"prometheus.yaml"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:'global:\n  scrape_interval: 15s\n  evaluation_interval: 15s\n\nrule_files:\n  # - "first.rules"\n  # - "second.rules"\n\nscrape_configs:\n  - job_name: prometheus\n    static_configs:\n      - targets: ["localhost:9090"]\n\n  - job_name: openc3-internal-metrics\n    metrics_path: "/openc3-api/internal/metrics"\n    static_configs:\n      - targets: ["openc3-cmd-tlm-api:2901"]\n\n  - job_name: openc3-cmd-tlm-api\n    metrics_path: "/openc3-api/metrics"\n    static_configs:\n      - targets: ["openc3-cmd-tlm-api:2901"]\n\n  - job_name: openc3-script-runner-api\n    metrics_path: "/script-api/metrics"\n    static_configs:\n      - targets: ["openc3-script-runner-api:2902"]\n\n  - job_name: minio-job\n    metrics_path: /minio/v2/metrics/cluster\n    scheme: http\n    static_configs:\n    - targets: [\'openc3-minio:9000\']\n'})}),"\n",(0,o.jsx)(n.p,{children:"Dockerfile"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"FROM prom/prometheus:v2.24.1\nADD prometheus.yaml /etc/prometheus/\n"})}),"\n",(0,o.jsx)(n.h3,{id:"grafana",children:(0,o.jsx)(n.a,{href:"https://grafana.com/",children:"Grafana"})}),"\n",(0,o.jsxs)(n.blockquote,{children:["\n",(0,o.jsx)(n.p,{children:"Grafana is a multi-platform open source analytics and interactive visualization web application. It provides charts, graphs, and alerts for the web when connected to supported data sources."}),"\n"]}),"\n",(0,o.jsx)(n.h4,{id:"notes-3",children:"Notes"}),"\n",(0,o.jsx)(n.p,{children:"datasource.yaml"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"apiVersion: 1\n\ndatasources:\n  - name: Prometheus\n    type: prometheus\n    # Access mode - proxy (server in the UI) or direct (browser in the UI).\n    access: proxy\n    url: http://openc3-prometheus:9090\n"})}),"\n",(0,o.jsx)(n.p,{children:"Dockerfile"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"FROM grafana/grafana\n\nCOPY datasource.yaml /etc/grafana/provisioning/datasources/\n"})})]})}function h(e={}){const{wrapper:n}={...(0,s.a)(),...e.components};return n?(0,o.jsx)(n,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},5392:(e,n,t)=>{t.d(n,{Z:()=>a,a:()=>i});var o=t(2784);const s={},r=o.createContext(s);function i(e){const n=o.useContext(r);return o.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:i(e.components),o.createElement(r.Provider,{value:n},e.children)}}}]);